public void SegmentBackgroundCompactionTest()
{
var segmentFactory = new MemorySegmentFactory <int>(10);
using (var queue = new DiskQueue <int>("dummy", segmentFactory, -1, true, 100))
{
Assert.AreEqual(1, queue.SegmentCount);
for (int i = 0; i < segmentFactory.Capacity; i++)
{
queue.Add(i);
}
Assert.AreEqual(1, queue.SegmentCount);
queue.Add(-1);
Assert.AreEqual(2, queue.SegmentCount);
Assert.AreEqual(2, segmentFactory.AllocatedSegments.Count);
for (int i = 0; i < segmentFactory.Capacity; i++)
{
queue.Take();
}
Assert.IsTrue(segmentFactory.AllocatedSegments[0].IsCompleted);
TimingAssert.AreEqual(10000, 1, () => { Interlocked.MemoryBarrier(); return(queue.SegmentCount); });
Assert.AreEqual(2, segmentFactory.AllocatedSegments.Count);
}
}
public void SegmentDiscoveryTest()
{
var segmentFactory = new MemorySegmentFactory <int>(10);
using (var queue = new DiskQueue <int>("dummy", segmentFactory, -1, false))
{
Assert.AreEqual(1, queue.SegmentCount);
for (int i = 0; i < segmentFactory.Capacity * 2; i++)
{
queue.Add(i);
}
Assert.AreEqual(2, queue.SegmentCount);
}
using (var queue = new DiskQueue <int>("dummy", segmentFactory, -1, false))
{
Assert.AreEqual(3, queue.SegmentCount);
Assert.AreEqual(3, segmentFactory.AllocatedSegments.Count);
for (int i = 0; i < segmentFactory.Capacity * 2; i++)
{
Assert.AreEqual(i, queue.Take());
}
Assert.AreEqual(2, queue.SegmentCount);
}
}
// =========================
private void PeekWakesUpTest(DiskQueue <int> queue)
{
Barrier bar = new Barrier(3);
AtomicNullableBool peekResult = new AtomicNullableBool();
AtomicNullableBool peekResult2 = new AtomicNullableBool();
Task task = Task.Run(() =>
{
bar.SignalAndWait();
int item = 0;
peekResult.Value = queue.TryPeek(out item, 60000);
Assert.AreEqual(100, item);
});
Task task2 = Task.Run(() =>
{
bar.SignalAndWait();
int item = 0;
peekResult2.Value = queue.TryPeek(out item, 60000);
Assert.AreEqual(100, item);
});
bar.SignalAndWait();
Thread.Sleep(10);
Assert.IsFalse(peekResult.HasValue);
Assert.IsFalse(peekResult2.HasValue);
queue.Add(100);
TimingAssert.AreEqual(10000, true, () => peekResult.Value);
TimingAssert.AreEqual(10000, true, () => peekResult2.Value);
Task.WaitAll(task, task2);
}
public void SegmentAllocationCompactionTest()
{
var segmentFactory = new MemorySegmentFactory <int>(10);
using (var queue = new DiskQueue <int>("dummy", segmentFactory, -1, false))
{
Assert.AreEqual(1, queue.SegmentCount);
for (int i = 0; i < segmentFactory.Capacity; i++)
{
queue.Add(i);
}
Assert.AreEqual(1, queue.SegmentCount);
queue.Add(-1);
Assert.AreEqual(2, queue.SegmentCount);
Assert.AreEqual(2, segmentFactory.AllocatedSegments.Count);
for (int i = 0; i < segmentFactory.Capacity; i++)
{
queue.Take();
}
Assert.IsTrue(segmentFactory.AllocatedSegments[0].IsCompleted);
// should trigger compaction
queue.Take();
Assert.AreEqual(1, queue.SegmentCount);
Assert.AreEqual(2, segmentFactory.AllocatedSegments.Count);
}
}
// =========================
private void TimeoutWorksTest(DiskQueue <int> queue)
{
Barrier bar = new Barrier(2);
AtomicNullableBool takeResult = new AtomicNullableBool();
AtomicNullableBool peekResult = new AtomicNullableBool();
AtomicNullableBool addResult = new AtomicNullableBool();
Task task = Task.Run(() =>
{
bar.SignalAndWait();
int item = 0;
takeResult.Value = queue.TryTake(out item, 100);
peekResult.Value = queue.TryPeek(out item, 100);
while (queue.TryAdd(-1))
{
;
}
addResult.Value = queue.TryAdd(100, 100);
});
bar.SignalAndWait();
TimingAssert.AreEqual(10000, false, () => takeResult.Value, "take");
TimingAssert.AreEqual(10000, false, () => peekResult.Value, "peek");
TimingAssert.AreEqual(10000, false, () => addResult.Value, "Add");
task.Wait();
}
// =========================
private void AddWakesUpTest(DiskQueue <int> queue, int segmentCapacity)
{
while (queue.TryAdd(100))
{
; // Fill queue
}
queue.AddForced(200);
Barrier bar = new Barrier(2);
AtomicNullableBool addResult = new AtomicNullableBool();
Task task = Task.Run(() =>
{
bar.SignalAndWait();
addResult.Value = queue.TryAdd(-100, 60000);
});
bar.SignalAndWait();
Thread.Sleep(10);
Assert.IsFalse(addResult.HasValue);
queue.Take();
Thread.Sleep(10);
Assert.IsFalse(addResult.HasValue);
for (int i = 0; i < segmentCapacity; i++)
{
int tmp = 0;
Assert.IsTrue(queue.TryTake(out tmp));
}
TimingAssert.AreEqual(10000, true, () => addResult.Value);
task.Wait();
}
// =========================
private void TakeWakesUpTest(DiskQueue <int> queue)
{
Barrier bar = new Barrier(2);
AtomicNullableBool takeResult = new AtomicNullableBool();
AtomicNullableBool takeResult2 = new AtomicNullableBool();
Task task = Task.Run(() =>
{
bar.SignalAndWait();
int item = 0;
takeResult.Value = queue.TryTake(out item, 60000);
Assert.AreEqual(100, item);
item = 0;
takeResult2.Value = queue.TryTake(out item, 60000);
Assert.AreEqual(200, item);
});
bar.SignalAndWait();
Thread.Sleep(10);
Assert.IsFalse(takeResult.HasValue);
queue.Add(100);
TimingAssert.AreEqual(10000, true, () => takeResult.Value);
Thread.Sleep(10);
Assert.IsFalse(takeResult2.HasValue);
queue.Add(200);
TimingAssert.AreEqual(10000, true, () => takeResult2.Value);
task.Wait();
}
public void TestConstruction()
{
var factory = new MemorySegmentFactory <int>(100);
using (var queue = new DiskQueue <int>("dummy", factory, 10, true, 500))
{
Assert.AreEqual("dummy", queue.Path);
Assert.AreEqual(0, queue.Count);
Assert.IsTrue(queue.IsEmpty);
Assert.IsTrue(queue.IsBackgroundCompactionEnabled);
Assert.AreEqual(1, queue.SegmentCount);
Assert.AreEqual(1, factory.AllocatedSegments.Count);
Assert.AreEqual(factory.Capacity * 10, queue.BoundedCapacity);
for (int i = 0; i < factory.Capacity + 1; i++)
{
queue.Add(1);
}
Assert.AreEqual(factory.Capacity + 1, queue.Count);
Assert.IsFalse(queue.IsEmpty);
Assert.AreEqual(2, queue.SegmentCount);
Assert.AreEqual(2, factory.AllocatedSegments.Count);
}
}
private static void RunMemCountingTest(int segmentCapacity, int segmentCount, bool backComp, Action <DiskQueue <int>, CountingMemorySegmentFactory <int> > testRunner)
{
var factory = new CountingMemorySegmentFactory <int>(segmentCapacity);
using (var q = new DiskQueue <int>("dummy", factory, segmentCount, backComp, 500))
testRunner(q, factory);
}
private static IQueue <long> CreateQueue1(string dir)
{
DiskQueue <int> disk = new DiskQueue <int>(dir, new NonPersistentDiskQueueSegmentFactory <int>(10000, "prefix", new ItemSerializer()), 100, true);
MemoryQueue <int> preDisk = new MemoryQueue <int>(5000);
LevelingQueue <int> diskWrap = new LevelingQueue <int>(preDisk, disk, LevelingQueueAddingMode.PreserveOrder, true);
TransformationQueue <long, int> transform = new TransformationQueue <long, int>(diskWrap, new TypeConverter());
MemoryQueue <long> topMem = new MemoryQueue <long>(1000);
LevelingQueue <long> topQ = new LevelingQueue <long>(topMem, transform, LevelingQueueAddingMode.PreserveOrder, true);
return(topQ);
}
// ==============================
private void TestSimpleAddTakePeek(DiskQueue <int> queue)
{
Assert.AreEqual(0, queue.Count);
Assert.IsTrue(queue.IsEmpty);
Assert.IsTrue(queue.TryAdd(1));
Assert.AreEqual(1, queue.Count);
Assert.IsFalse(queue.IsEmpty);
int item = 0;
Assert.IsTrue(queue.TryPeek(out item));
Assert.AreEqual(1, item);
Assert.IsFalse(queue.IsEmpty);
item = 0;
Assert.IsTrue(queue.TryTake(out item));
Assert.AreEqual(1, item);
Assert.AreEqual(0, queue.Count);
Assert.IsTrue(queue.IsEmpty);
for (int i = 1; i < 1000; i++)
{
if (i % 2 == 0)
{
queue.Add(i);
}
else
{
queue.AddForced(i);
}
Assert.AreEqual(i, queue.Count);
}
int itemNum = 0;
while (queue.TryTake(out item))
{
itemNum++;
Assert.AreEqual(itemNum, item);
}
Assert.AreEqual(999, itemNum);
Assert.AreEqual(0, queue.Count);
Assert.IsTrue(queue.IsEmpty);
}
private static TimeSpan RunConcurrentDiskQFile(string name, int elemCount, int addThCount, int takeThCount, int addSpin, int takeSpin)
{
if (Directory.Exists("dummy"))
{
Directory.Delete("dummy", true);
}
Directory.CreateDirectory("dummy");
DiskQueue <int> col = new DiskQueue <int>("dummy",
new NonPersistentDiskQueueSegmentFactory <int>(10000, "prefix", new ItemSerializer(), 256, 16),
10, true);
//DiskQueue<int> col = new DiskQueue<int>("dummy",
// new PersistentDiskQueueSegmentFactory<int>(10000, "prefix", new ItemSerializer(), false, false, 1000, 64),
// 10, true);
CancellationTokenSource srcCancel = new CancellationTokenSource();
Thread[] addThreads = new Thread[addThCount];
Thread[] takeThreads = new Thread[takeThCount];
int addedElemCount = 0;
List <int> globalList = new List <int>();
Barrier barierStart = new Barrier(1 + addThreads.Length + takeThreads.Length);
Barrier barierAdders = new Barrier(1 + addThreads.Length);
Barrier barierTakers = new Barrier(1 + takeThreads.Length);
Action addAction = () =>
{
barierStart.SignalAndWait();
int index = 0;
while ((index = Interlocked.Increment(ref addedElemCount)) <= elemCount)
{
col.Add(index - 1);
Thread.SpinWait(addSpin);
}
barierAdders.SignalAndWait();
};
Action takeAction = () =>
{
CancellationToken myToken = srcCancel.Token;
List <int> valList = new List <int>(elemCount / takeThCount + 100);
barierStart.SignalAndWait();
try
{
while (!srcCancel.IsCancellationRequested)
{
int val = 0;
val = col.Take(myToken);
valList.Add(val);
Thread.SpinWait(takeSpin);
}
}
catch (OperationCanceledException)
{
}
int val2 = 0;
while (col.TryTake(out val2))
{
valList.Add(val2);
}
barierTakers.SignalAndWait();
lock (globalList)
{
globalList.AddRange(valList);
}
};
for (int i = 0; i < addThreads.Length; i++)
{
addThreads[i] = new Thread(new ThreadStart(addAction));
}
for (int i = 0; i < takeThreads.Length; i++)
{
takeThreads[i] = new Thread(new ThreadStart(takeAction));
}
for (int i = 0; i < takeThreads.Length; i++)
{
takeThreads[i].Start();
}
for (int i = 0; i < addThreads.Length; i++)
{
addThreads[i].Start();
}
barierStart.SignalAndWait();
Stopwatch sw = Stopwatch.StartNew();
barierAdders.SignalAndWait();
srcCancel.Cancel();
barierTakers.SignalAndWait();
sw.Stop();
for (int i = 0; i < addThreads.Length; i++)
{
addThreads[i].Join();
}
for (int i = 0; i < takeThreads.Length; i++)
{
takeThreads[i].Join();
}
globalList.Sort();
if (globalList.Count != elemCount)
{
Console.WriteLine("Bad count");
}
for (int i = 0; i < globalList.Count; i++)
{
if (globalList[i] != i)
{
Console.WriteLine("invalid elements");
break;
}
}
if (name != null)
{
Console.WriteLine(name + ". DiskQFile. Time = " + sw.ElapsedMilliseconds.ToString() + "ms");
}
col.Dispose();
return(sw.Elapsed);
}
// ======================
private void PreserveOrderTest(DiskQueue <int> queue, int elemCount)
{
Barrier bar = new Barrier(2);
CancellationTokenSource cancelled = new CancellationTokenSource();
List <int> takenElems = new List <int>(elemCount + 1);
Action addAction = () =>
{
int curElem = 0;
Random rnd = new Random(Environment.TickCount + Thread.CurrentThread.ManagedThreadId);
bar.SignalAndWait();
while (curElem < elemCount)
{
if (rnd.Next(100) == 0)
{
queue.AddForced(curElem);
}
else
{
queue.Add(curElem);
}
if (rnd.Next(100) == 0)
{
Thread.Yield();
}
SpinWaitHelper.SpinWait(rnd.Next(12));
curElem++;
}
cancelled.Cancel();
};
Action takeAction = () =>
{
Random rnd = new Random(Environment.TickCount + Thread.CurrentThread.ManagedThreadId);
bar.SignalAndWait();
try
{
while (!cancelled.IsCancellationRequested)
{
takenElems.Add(queue.Take(cancelled.Token));
if (rnd.Next(100) == 0)
{
Thread.Yield();
}
SpinWaitHelper.SpinWait(rnd.Next(12));
}
}
catch (OperationCanceledException) { }
int item = 0;
while (queue.TryTake(out item))
{
takenElems.Add(item);
}
};
var sw = System.Diagnostics.Stopwatch.StartNew();
Task addTask = Task.Factory.StartNew(addAction, TaskCreationOptions.LongRunning);
Task takeTask = Task.Factory.StartNew(takeAction, TaskCreationOptions.LongRunning);
Task.WaitAll(addTask, takeTask);
Assert.AreEqual(elemCount, takenElems.Count);
for (int i = 0; i < takenElems.Count; i++)
{
if (i != takenElems[i])
{
Assert.AreEqual(i, takenElems[i], $"i != takenElems[i], nextItem = {takenElems[i + 1]}");
}
}
}
// =========================
private void CancellationWorksTest(DiskQueue <int> queue)
{
Barrier bar = new Barrier(2);
CancellationTokenSource takeSource = new CancellationTokenSource();
AtomicInt takeResult = new AtomicInt();
CancellationTokenSource peekSource = new CancellationTokenSource();
AtomicInt peekResult = new AtomicInt();
CancellationTokenSource addSource = new CancellationTokenSource();
AtomicInt addResult = new AtomicInt();
Task task = Task.Run(() =>
{
bar.SignalAndWait();
int item = 0;
try
{
takeResult.Value = queue.TryTake(out item, 60000, takeSource.Token) ? 1 : -1;
}
catch (OperationCanceledException)
{
takeResult.Value = 3;
}
try
{
peekResult.Value = queue.TryPeek(out item, 60000, peekSource.Token) ? 1 : -1;
}
catch (OperationCanceledException)
{
peekResult.Value = 3;
}
while (queue.TryAdd(-1))
{
;
}
try
{
addResult.Value = queue.TryAdd(100, 60000, addSource.Token) ? 1 : -1;
}
catch (OperationCanceledException)
{
addResult.Value = 3;
}
});
bar.SignalAndWait();
Thread.Sleep(10);
Assert.AreEqual(0, takeResult);
takeSource.Cancel();
TimingAssert.AreEqual(10000, 3, () => takeResult);
Thread.Sleep(10);
Assert.AreEqual(0, peekResult);
peekSource.Cancel();
TimingAssert.AreEqual(10000, 3, () => peekResult);
Thread.Sleep(10);
Assert.AreEqual(0, addResult);
addSource.Cancel();
TimingAssert.AreEqual(10000, 3, () => addResult);
task.Wait();
}
// ==================================
private void RunComplexTest(DiskQueue <int> q, int elemCount, int thCount)
{
int atomicRandom = 0;
int trackElemCount = elemCount;
int addFinished = 0;
Thread[] threadsTake = new Thread[thCount];
Thread[] threadsAdd = new Thread[thCount];
CancellationTokenSource tokSrc = new CancellationTokenSource();
List <int> global = new List <int>(elemCount);
Action addAction = () =>
{
Random rnd = new Random(Environment.TickCount + Interlocked.Increment(ref atomicRandom) * thCount * 2);
while (true)
{
int item = Interlocked.Decrement(ref trackElemCount);
if (item < 0)
{
break;
}
if (rnd.Next(100) == 0)
{
q.AddForced(item);
}
else
{
q.Add(item);
}
int sleepTime = rnd.Next(12);
int tmpItem = 0;
if (q.TryPeek(out tmpItem) && tmpItem == item)
{
sleepTime += 15;
}
if (sleepTime > 0)
{
SpinWaitHelper.SpinWait(sleepTime);
}
}
Interlocked.Increment(ref addFinished);
};
Action takeAction = () =>
{
Random rnd = new Random(Environment.TickCount + Interlocked.Increment(ref atomicRandom) * thCount * 2);
List <int> data = new List <int>();
try
{
while (Volatile.Read(ref addFinished) < thCount)
{
int tmp = 0;
if (q.TryTake(out tmp, -1, tokSrc.Token))
{
data.Add((int)tmp);
}
int sleepTime = rnd.Next(12);
if (sleepTime > 0)
{
SpinWaitHelper.SpinWait(sleepTime);
}
}
}
catch (OperationCanceledException) { }
int tmp2;
while (q.TryTake(out tmp2))
{
data.Add((int)tmp2);
}
lock (global)
global.AddRange(data);
};
for (int i = 0; i < threadsTake.Length; i++)
{
threadsTake[i] = new Thread(new ThreadStart(takeAction));
}
for (int i = 0; i < threadsAdd.Length; i++)
{
threadsAdd[i] = new Thread(new ThreadStart(addAction));
}
for (int i = 0; i < threadsTake.Length; i++)
{
threadsTake[i].Start();
}
for (int i = 0; i < threadsAdd.Length; i++)
{
threadsAdd[i].Start();
}
for (int i = 0; i < threadsAdd.Length; i++)
{
threadsAdd[i].Join();
}
tokSrc.Cancel();
for (int i = 0; i < threadsTake.Length; i++)
{
threadsTake[i].Join();
}
Assert.AreEqual(elemCount, global.Count);
global.Sort();
for (int i = 0; i < elemCount; i++)
{
Assert.AreEqual(i, global[i]);
}
}
// ======================
private void ValidateCountTest(DiskQueue <int> queue, CommonSegmentFactory <int> factory, int elemCount)
{
Barrier bar = new Barrier(2);
CancellationTokenSource cancelled = new CancellationTokenSource();
List <int> takenElems = new List <int>(elemCount + 1);
AtomicBool needSync = new AtomicBool();
Action addAction = () =>
{
int curElem = 0;
Random rnd = new Random(Environment.TickCount + Thread.CurrentThread.ManagedThreadId);
bar.SignalAndWait();
while (curElem < elemCount)
{
bool itemAdded = true;
if (rnd.Next(100) == 0)
{
queue.AddForced(curElem);
}
else if (needSync.Value)
{
itemAdded = queue.TryAdd(curElem);
}
else
{
queue.Add(curElem);
}
if (rnd.Next(100) == 0)
{
Thread.Yield();
}
SpinWaitHelper.SpinWait(rnd.Next(12));
if (itemAdded)
{
curElem++;
}
Assert.IsTrue(itemAdded || needSync.Value);
if (curElem % 1000 == 0)
{
needSync.Value = true;
}
if (needSync.Value && bar.ParticipantsRemaining == 1)
{
Assert.AreEqual(factory.SumCountFromAllocated(), queue.Count);
needSync.Value = false;
bar.SignalAndWait();
}
}
cancelled.Cancel();
};
Action takeAction = () =>
{
Random rnd = new Random(Environment.TickCount + Thread.CurrentThread.ManagedThreadId);
bar.SignalAndWait();
try
{
while (!cancelled.IsCancellationRequested)
{
takenElems.Add(queue.Take(cancelled.Token));
if (rnd.Next(100) == 0)
{
Thread.Yield();
}
SpinWaitHelper.SpinWait(rnd.Next(12));
if (needSync.Value)
{
bar.SignalAndWait(cancelled.Token);
}
}
}
catch (OperationCanceledException) { }
int item = 0;
while (queue.TryTake(out item))
{
takenElems.Add(item);
}
};
var sw = System.Diagnostics.Stopwatch.StartNew();
Task addTask = Task.Factory.StartNew(addAction, TaskCreationOptions.LongRunning);
Task takeTask = Task.Factory.StartNew(takeAction, TaskCreationOptions.LongRunning);
Task.WaitAll(addTask, takeTask);
Assert.AreEqual(elemCount, takenElems.Count);
for (int i = 0; i < takenElems.Count; i++)
{
if (i != takenElems[i])
{
Assert.AreEqual(i, takenElems[i], $"i != takenElems[i], nextItem = {takenElems[i + 1]}");
}
}
}
