public void TestDiscoverIndex()
{
var serializer = new BinarySerializer <MockClass>(BinarySerializerTest.MockSerializer, BinarySerializerTest.MockDeserializer);
var logger = new MemoryLogger("TestDiscoverIndex");
var queue = new FilePersistentQueue <MockClass>(1000, _queueDirectory, serializer, _dataFileProvider, logger);
LoadQueue(queue);
// Increment the tail to 20 to simulate the first records being bad ones.
// This is testing the function where only the last item in a consecutive sequence from the tail are processed.
queue.Tail = 20;
LoadQueue(queue);
var indexFile = Path.Combine(_appDataDir, queue.QueueDirectory, "Index");
File.Delete(indexFile);
queue.Head = -10;
queue.Tail = -10;
queue.DiscoverIndex();
Assert.Equal(30, queue.Tail);
Assert.Equal(20, queue.Head);
Assert.True(queue.TryDequeue(out _));
Assert.Equal(21, queue.Head);
}
public void TestLoadIndex(string contents, bool isValid)
{
var serializer = new BinarySerializer <MockClass>(BinarySerializerTest.MockSerializer, BinarySerializerTest.MockDeserializer);
var logger = new MemoryLogger("TestLoadIndex");
var queue = new FilePersistentQueue <MockClass>(1000, _queueDirectory, serializer, _dataFileProvider, logger);
LoadQueue(queue);
var index = Path.Combine(_appDataDir, queue.QueueDirectory, "Index");
File.WriteAllText(index, contents);
Assert.Equal(isValid, queue.LoadIndex());
if (isValid)
{
Assert.Equal(101, queue.Head);
Assert.Equal(202, queue.Tail);
Assert.DoesNotContain(logger.Entries, i => i.Contains("ERROR", StringComparison.CurrentCultureIgnoreCase));
}
else
{
Assert.Equal(0, queue.Head);
Assert.Equal(10, queue.Tail);
Assert.Contains(logger.Entries, i => i.Contains("Error", StringComparison.CurrentCultureIgnoreCase));
}
}
public void TestFilePersistentQueue()
{
var serializer = new BinarySerializer <MockClass>(BinarySerializerTest.MockSerializer, BinarySerializerTest.MockDeserializer);
var directory = Path.Combine(QueueDirectory, "TestFilePersistentQueue");
if (Directory.Exists(directory))
{
Directory.Delete(directory, true);
}
var queue = new FilePersistentQueue <MockClass>(1000, directory, serializer);
var list = BinarySerializerTest.CreateList().Where(i => queue.TryEnqueue(i)).ToList();
Assert.Equal(list.Count, queue.Count);
Assert.Equal(list.Count, Directory.GetFiles(directory, "0*").Count());
var list2 = new List <MockClass>();
while (queue.TryDequeue(out var item))
{
list2.Add(item);
}
Assert.True(list.SequenceEqual(list2, new MockClassComparer()));
Assert.Equal(0, queue.Count);
Assert.Empty(Directory.GetFiles(directory, "0*"));
}
public void TestEmptyQueue()
{
FilePersistentQueue <int> queue = CreateQueue("empty");
Assert.Throws <InvalidOperationException>(() =>
{
int i = queue.Dequeue();
});
}
public void TestQueueCapacity()
{
FilePersistentQueue <int> queue = CreateQueue("capacity");
Assert.Throws <InvalidOperationException>(() =>
{
for (int i = 0; i < 11; i++)
{
queue.Enqueue(i);
}
});
}
private static void LoadQueue(FilePersistentQueue <MockClass> queue)
{
var random = Utility.Random;
for (int i = 0; i < 10; i++)
{
queue.TryEnqueue(new MockClass
{
AnInt = random.Next(),
ALong = random.Next(),
ADateTime = DateTime.Now,
AString = TestUtility.RandomString(1000),
AMemortySteam = Utility.StringToStream(TestUtility.RandomString(1000))
});
}
}
public void TestHighLowBufferWithPersistentQueue()
{
List <int> output = new List <int>();
ManualResetEvent sinkWaitHandle = new ManualResetEvent(false);
var dataDir = Path.Combine(AppContext.BaseDirectory, Guid.NewGuid().ToString());
var queueDir = Path.Combine(dataDir, "HighLowTest");
Directory.CreateDirectory(queueDir);
if (Directory.Exists(queueDir))
{
Directory.Delete(queueDir, true);
}
BinarySerializer <int> integerSerializer = new BinarySerializer <int>(
(w, i) => w.Write(i),
(r) => r.ReadInt32());
FilePersistentQueue <int> queue = new FilePersistentQueue <int>(
100,
queueDir,
integerSerializer,
new ProtectedAppDataFileProvider(dataDir),
NullLogger.Instance);
HiLowBuffer <int> buffer = new HiLowBuffer <int>(1, null, l =>
{
sinkWaitHandle.WaitOne();
output.Add(l);
}, queue);
Task.Run(() =>
{
for (int i = 0; i < 2; i++)
{
buffer.Add(i);
}
});
//should block
Thread.Sleep(500);
buffer.Requeue(-1, false);
sinkWaitHandle.Set();
Thread.Sleep(500);
Assert.Equal(3, output.Count);
Assert.Equal(-1, output[2]); //Requeue item come out last
Assert.True(File.Exists(Path.Combine(queueDir, "index")));
}
public async Task PersistentQueue_PushAndGet(int batchCount)
{
var dataDir = Path.Combine(AppContext.BaseDirectory, Guid.NewGuid().ToString());
var queueDir = Path.Combine(dataDir, _queueDirName);
Directory.CreateDirectory(queueDir);
var fileProvider = new ProtectedAppDataFileProvider(dataDir);
try
{
var secondary = new FilePersistentQueue <List <int> >(100, queueDir, new IntegerListSerializer(), fileProvider, NullLogger.Instance);
var q = new AsyncBatchQueue <int>(1000,
new long[] { batchCount },
new Func <int, long>[] { s => 1 }, secondary);
await q.PushSecondaryAsync(Enumerable.Range(0, batchCount + 1).ToList());
var output = new List <int>();
var getTask = q.GetNextBatchAsync(output, 10 * 1000).AsTask();
Assert.True(getTask.Wait(1000));
Assert.Equal(batchCount, output.Count);
if (batchCount == 0)
{
return;
}
// second call should return the last item
output.Clear();
await q.GetNextBatchAsync(output, 100);
Assert.Single(output);
}
finally
{
if (Directory.Exists(dataDir))
{
Directory.Delete(dataDir, true);
}
}
}
public void TestFilePersistentQueue()
{
var serializer = new BinarySerializer <MockClass>(BinarySerializerTest.MockSerializer, BinarySerializerTest.MockDeserializer);
var queue = new FilePersistentQueue <MockClass>(1000, _queueDirectory, serializer, _dataFileProvider, NullLogger.Instance);
var list = BinarySerializerTest.CreateList().Where(i => queue.TryEnqueue(i)).ToList();
Assert.Equal(list.Count, queue.Count);
Assert.Equal(list.Count, Directory.GetFiles(Path.Combine(_appDataDir, _queueDirectory), "0*").Count());
var list2 = new List <MockClass>();
while (queue.TryDequeue(out var item))
{
list2.Add(item);
}
Assert.True(list.SequenceEqual(list2, new MockClassComparer()));
Assert.Equal(0, queue.Count);
Assert.Empty(Directory.GetFiles(Path.Combine(_appDataDir, _queueDirectory), "0*"));
}
public async Task PersistentQueue_PullBothQueues()
{
var dataDir = Path.Combine(AppContext.BaseDirectory, Guid.NewGuid().ToString());
var queueDir = Path.Combine(dataDir, _queueDirName);
Directory.CreateDirectory(queueDir);
var fileProvider = new ProtectedAppDataFileProvider(dataDir);
try
{
var secondary = new FilePersistentQueue <List <int> >(10, queueDir, new IntegerListSerializer(), fileProvider, NullLogger.Instance);
var q = new AsyncBatchQueue <int>(500,
new long[] { 500 },
new Func <int, long>[] { s => 1 }, secondary);
await q.PushSecondaryAsync(Enumerable.Range(0, 500).ToList());
for (var i = 500; i < 1000; i++)
{
await q.PushAsync(i);
}
// pull 3 times
var output = new List <int>();
await q.GetNextBatchAsync(output, 1000);
await q.GetNextBatchAsync(output, 1000);
await q.GetNextBatchAsync(output, 1000);
Assert.Equal(1000, output.Distinct().Count());
}
finally
{
if (Directory.Exists(dataDir))
{
Directory.Delete(dataDir, true);
}
}
}
public void TestHighLowBufferWithPersistentQueue()
{
List <int> output = new List <int>();
ManualResetEvent sinkWaitHandle = new ManualResetEvent(false);
string directory = Path.Combine(FilePersistenceQueueTest.QueueDirectory, "HighLowTest");
if (Directory.Exists(directory))
{
Directory.Delete(directory, true);
}
BinarySerializer <int> integerSerializer = new BinarySerializer <int>(
(w, i) => w.Write(i),
(r) => r.ReadInt32());
FilePersistentQueue <int> queue = new FilePersistentQueue <int>(
100,
directory,
integerSerializer);
HiLowBuffer <int> buffer = new HiLowBuffer <int>(1, l =>
{
sinkWaitHandle.WaitOne();
output.Add(l);
}, queue);
Task.Run(() =>
{
for (int i = 0; i < 2; i++)
{
buffer.Add(i);
}
});
//shoud block
Thread.Sleep(500);
buffer.Requeue(-1, false);
sinkWaitHandle.Set();
Thread.Sleep(500);
Assert.Equal(3, output.Count);
Assert.Equal(-1, output[2]); //Requeue item come out last
Assert.True(File.Exists(Path.Combine(directory, "index")));
}
public void TestFilePersistentQueue()
{
BinarySerializer <MockClass> serializer = new BinarySerializer <MockClass>(
BinarySerializerTest.MockSerializer,
BinarySerializerTest.MockDeserializer
);
string directory = Path.Combine(QueueDirectory, "TestFilePersistentQueue");
if (Directory.Exists(directory))
{
Directory.Delete(directory, true);
}
FilePersistentQueue <MockClass> queue = new FilePersistentQueue <MockClass>(
1000,
directory,
serializer
);
List <MockClass> list = BinarySerializerTest.CreateList();
foreach (var item in list)
{
queue.Enqueue(item);
}
Assert.Equal(list.Count, queue.Count);
Assert.Equal(list.Count, Directory.GetFiles(directory, "0*").Count());
List <MockClass> list2 = new List <MockClass>();
while (queue.Count > 0)
{
list2.Add(queue.Dequeue());
}
Assert.True(list.SequenceEqual(list2, new MockClassComparer()));
Assert.Equal(0, queue.Count);
Assert.Empty(Directory.GetFiles(directory, "0*"));
}
public async Task PersistentQueue_ConcurrentRead(int readerCount, int itemCount)
{
var dataDir = Path.Combine(AppContext.BaseDirectory, Guid.NewGuid().ToString());
var queueDir = Path.Combine(dataDir, _queueDirName);
Directory.CreateDirectory(queueDir);
var fileProvider = new ProtectedAppDataFileProvider(dataDir);
try
{
var secondary = new FilePersistentQueue <List <int> >(100000, queueDir, new IntegerListSerializer(), fileProvider, NullLogger.Instance);
using var cts = new CancellationTokenSource();
using var semaphore = new SemaphoreSlim(0, readerCount);
var results = new List <int>();
var q = new AsyncBatchQueue <int>(10000,
new long[] { 100 },
new Func <int, long>[] { s => 1 }, secondary);
async Task readerTask()
{
var output = new List <int>();
await semaphore.WaitAsync();
// we're trying to test that the readers will 'eventually' read all the items, so we do several pulls here
await q.GetNextBatchAsync(output, 500);
await q.GetNextBatchAsync(output, 500);
await q.GetNextBatchAsync(output, 500);
await Task.Delay(100);
lock (results)
{
results.AddRange(output);
}
};
var readers = new Task[readerCount];
for (var i = 0; i < readerCount; i++)
{
readers[i] = readerTask();
}
for (var i = 0; i < itemCount; i++)
{
if (i % 2 == 0)
{
await q.PushAsync(i);
}
else
{
await q.PushSecondaryAsync(new List <int> {
i
});
}
}
semaphore.Release(readerCount);
await Task.WhenAll(readers);
_output.WriteLine(results.Count.ToString());
_output.WriteLine(q.EstimateSize().ToString());
_output.WriteLine(q.EstimateSecondaryQueueSize().ToString());
Assert.Equal(itemCount, results.Distinct().Count());
}
finally
{
if (Directory.Exists(dataDir))
{
Directory.Delete(dataDir, true);
}
}
}