public void SegmentAllocationCompactionTest()
{
var segmentFactory = new MemorySegmentFactory <int>(10);
using (var queue = new DiskQueue <int>("dummy", segmentFactory, -1, false))
{
Assert.AreEqual(1, queue.SegmentCount);
for (int i = 0; i < segmentFactory.Capacity; i++)
{
queue.Add(i);
}
Assert.AreEqual(1, queue.SegmentCount);
queue.Add(-1);
Assert.AreEqual(2, queue.SegmentCount);
Assert.AreEqual(2, segmentFactory.AllocatedSegments.Count);
for (int i = 0; i < segmentFactory.Capacity; i++)
{
queue.Take();
}
Assert.IsTrue(segmentFactory.AllocatedSegments[0].IsCompleted);
// should trigger compaction
queue.Take();
Assert.AreEqual(1, queue.SegmentCount);
Assert.AreEqual(2, segmentFactory.AllocatedSegments.Count);
}
}
public void SegmentDiscoveryTest()
{
var segmentFactory = new MemorySegmentFactory <int>(10);
using (var queue = new DiskQueue <int>("dummy", segmentFactory, -1, false))
{
Assert.AreEqual(1, queue.SegmentCount);
for (int i = 0; i < segmentFactory.Capacity * 2; i++)
{
queue.Add(i);
}
Assert.AreEqual(2, queue.SegmentCount);
}
using (var queue = new DiskQueue <int>("dummy", segmentFactory, -1, false))
{
Assert.AreEqual(3, queue.SegmentCount);
Assert.AreEqual(3, segmentFactory.AllocatedSegments.Count);
for (int i = 0; i < segmentFactory.Capacity * 2; i++)
{
Assert.AreEqual(i, queue.Take());
}
Assert.AreEqual(2, queue.SegmentCount);
}
}
public void SegmentBackgroundCompactionTest()
{
var segmentFactory = new MemorySegmentFactory <int>(10);
using (var queue = new DiskQueue <int>("dummy", segmentFactory, -1, true, 100))
{
Assert.AreEqual(1, queue.SegmentCount);
for (int i = 0; i < segmentFactory.Capacity; i++)
{
queue.Add(i);
}
Assert.AreEqual(1, queue.SegmentCount);
queue.Add(-1);
Assert.AreEqual(2, queue.SegmentCount);
Assert.AreEqual(2, segmentFactory.AllocatedSegments.Count);
for (int i = 0; i < segmentFactory.Capacity; i++)
{
queue.Take();
}
Assert.IsTrue(segmentFactory.AllocatedSegments[0].IsCompleted);
TimingAssert.AreEqual(10000, 1, () => { Interlocked.MemoryBarrier(); return(queue.SegmentCount); });
Assert.AreEqual(2, segmentFactory.AllocatedSegments.Count);
}
}
// =========================
private void AddWakesUpTest(DiskQueue <int> queue, int segmentCapacity)
{
while (queue.TryAdd(100))
{
; // Fill queue
}
queue.AddForced(200);
Barrier bar = new Barrier(2);
AtomicNullableBool addResult = new AtomicNullableBool();
Task task = Task.Run(() =>
{
bar.SignalAndWait();
addResult.Value = queue.TryAdd(-100, 60000);
});
bar.SignalAndWait();
Thread.Sleep(10);
Assert.IsFalse(addResult.HasValue);
queue.Take();
Thread.Sleep(10);
Assert.IsFalse(addResult.HasValue);
for (int i = 0; i < segmentCapacity; i++)
{
int tmp = 0;
Assert.IsTrue(queue.TryTake(out tmp));
}
TimingAssert.AreEqual(10000, true, () => addResult.Value);
task.Wait();
}
private static TimeSpan RunConcurrentDiskQFile(string name, int elemCount, int addThCount, int takeThCount, int addSpin, int takeSpin)
{
if (Directory.Exists("dummy"))
{
Directory.Delete("dummy", true);
}
Directory.CreateDirectory("dummy");
DiskQueue <int> col = new DiskQueue <int>("dummy",
new NonPersistentDiskQueueSegmentFactory <int>(10000, "prefix", new ItemSerializer(), 256, 16),
10, true);
//DiskQueue<int> col = new DiskQueue<int>("dummy",
// new PersistentDiskQueueSegmentFactory<int>(10000, "prefix", new ItemSerializer(), false, false, 1000, 64),
// 10, true);
CancellationTokenSource srcCancel = new CancellationTokenSource();
Thread[] addThreads = new Thread[addThCount];
Thread[] takeThreads = new Thread[takeThCount];
int addedElemCount = 0;
List <int> globalList = new List <int>();
Barrier barierStart = new Barrier(1 + addThreads.Length + takeThreads.Length);
Barrier barierAdders = new Barrier(1 + addThreads.Length);
Barrier barierTakers = new Barrier(1 + takeThreads.Length);
Action addAction = () =>
{
barierStart.SignalAndWait();
int index = 0;
while ((index = Interlocked.Increment(ref addedElemCount)) <= elemCount)
{
col.Add(index - 1);
Thread.SpinWait(addSpin);
}
barierAdders.SignalAndWait();
};
Action takeAction = () =>
{
CancellationToken myToken = srcCancel.Token;
List <int> valList = new List <int>(elemCount / takeThCount + 100);
barierStart.SignalAndWait();
try
{
while (!srcCancel.IsCancellationRequested)
{
int val = 0;
val = col.Take(myToken);
valList.Add(val);
Thread.SpinWait(takeSpin);
}
}
catch (OperationCanceledException)
{
}
int val2 = 0;
while (col.TryTake(out val2))
{
valList.Add(val2);
}
barierTakers.SignalAndWait();
lock (globalList)
{
globalList.AddRange(valList);
}
};
for (int i = 0; i < addThreads.Length; i++)
{
addThreads[i] = new Thread(new ThreadStart(addAction));
}
for (int i = 0; i < takeThreads.Length; i++)
{
takeThreads[i] = new Thread(new ThreadStart(takeAction));
}
for (int i = 0; i < takeThreads.Length; i++)
{
takeThreads[i].Start();
}
for (int i = 0; i < addThreads.Length; i++)
{
addThreads[i].Start();
}
barierStart.SignalAndWait();
Stopwatch sw = Stopwatch.StartNew();
barierAdders.SignalAndWait();
srcCancel.Cancel();
barierTakers.SignalAndWait();
sw.Stop();
for (int i = 0; i < addThreads.Length; i++)
{
addThreads[i].Join();
}
for (int i = 0; i < takeThreads.Length; i++)
{
takeThreads[i].Join();
}
globalList.Sort();
if (globalList.Count != elemCount)
{
Console.WriteLine("Bad count");
}
for (int i = 0; i < globalList.Count; i++)
{
if (globalList[i] != i)
{
Console.WriteLine("invalid elements");
break;
}
}
if (name != null)
{
Console.WriteLine(name + ". DiskQFile. Time = " + sw.ElapsedMilliseconds.ToString() + "ms");
}
col.Dispose();
return(sw.Elapsed);
}
// ======================
private void PreserveOrderTest(DiskQueue <int> queue, int elemCount)
{
Barrier bar = new Barrier(2);
CancellationTokenSource cancelled = new CancellationTokenSource();
List <int> takenElems = new List <int>(elemCount + 1);
Action addAction = () =>
{
int curElem = 0;
Random rnd = new Random(Environment.TickCount + Thread.CurrentThread.ManagedThreadId);
bar.SignalAndWait();
while (curElem < elemCount)
{
if (rnd.Next(100) == 0)
{
queue.AddForced(curElem);
}
else
{
queue.Add(curElem);
}
if (rnd.Next(100) == 0)
{
Thread.Yield();
}
SpinWaitHelper.SpinWait(rnd.Next(12));
curElem++;
}
cancelled.Cancel();
};
Action takeAction = () =>
{
Random rnd = new Random(Environment.TickCount + Thread.CurrentThread.ManagedThreadId);
bar.SignalAndWait();
try
{
while (!cancelled.IsCancellationRequested)
{
takenElems.Add(queue.Take(cancelled.Token));
if (rnd.Next(100) == 0)
{
Thread.Yield();
}
SpinWaitHelper.SpinWait(rnd.Next(12));
}
}
catch (OperationCanceledException) { }
int item = 0;
while (queue.TryTake(out item))
{
takenElems.Add(item);
}
};
var sw = System.Diagnostics.Stopwatch.StartNew();
Task addTask = Task.Factory.StartNew(addAction, TaskCreationOptions.LongRunning);
Task takeTask = Task.Factory.StartNew(takeAction, TaskCreationOptions.LongRunning);
Task.WaitAll(addTask, takeTask);
Assert.AreEqual(elemCount, takenElems.Count);
for (int i = 0; i < takenElems.Count; i++)
{
if (i != takenElems[i])
{
Assert.AreEqual(i, takenElems[i], $"i != takenElems[i], nextItem = {takenElems[i + 1]}");
}
}
}
// ======================
private void ValidateCountTest(DiskQueue <int> queue, CommonSegmentFactory <int> factory, int elemCount)
{
Barrier bar = new Barrier(2);
CancellationTokenSource cancelled = new CancellationTokenSource();
List <int> takenElems = new List <int>(elemCount + 1);
AtomicBool needSync = new AtomicBool();
Action addAction = () =>
{
int curElem = 0;
Random rnd = new Random(Environment.TickCount + Thread.CurrentThread.ManagedThreadId);
bar.SignalAndWait();
while (curElem < elemCount)
{
bool itemAdded = true;
if (rnd.Next(100) == 0)
{
queue.AddForced(curElem);
}
else if (needSync.Value)
{
itemAdded = queue.TryAdd(curElem);
}
else
{
queue.Add(curElem);
}
if (rnd.Next(100) == 0)
{
Thread.Yield();
}
SpinWaitHelper.SpinWait(rnd.Next(12));
if (itemAdded)
{
curElem++;
}
Assert.IsTrue(itemAdded || needSync.Value);
if (curElem % 1000 == 0)
{
needSync.Value = true;
}
if (needSync.Value && bar.ParticipantsRemaining == 1)
{
Assert.AreEqual(factory.SumCountFromAllocated(), queue.Count);
needSync.Value = false;
bar.SignalAndWait();
}
}
cancelled.Cancel();
};
Action takeAction = () =>
{
Random rnd = new Random(Environment.TickCount + Thread.CurrentThread.ManagedThreadId);
bar.SignalAndWait();
try
{
while (!cancelled.IsCancellationRequested)
{
takenElems.Add(queue.Take(cancelled.Token));
if (rnd.Next(100) == 0)
{
Thread.Yield();
}
SpinWaitHelper.SpinWait(rnd.Next(12));
if (needSync.Value)
{
bar.SignalAndWait(cancelled.Token);
}
}
}
catch (OperationCanceledException) { }
int item = 0;
while (queue.TryTake(out item))
{
takenElems.Add(item);
}
};
var sw = System.Diagnostics.Stopwatch.StartNew();
Task addTask = Task.Factory.StartNew(addAction, TaskCreationOptions.LongRunning);
Task takeTask = Task.Factory.StartNew(takeAction, TaskCreationOptions.LongRunning);
Task.WaitAll(addTask, takeTask);
Assert.AreEqual(elemCount, takenElems.Count);
for (int i = 0; i < takenElems.Count; i++)
{
if (i != takenElems[i])
{
Assert.AreEqual(i, takenElems[i], $"i != takenElems[i], nextItem = {takenElems[i + 1]}");
}
}
}
