private void btnPersonDelete_Click(object sender, EventArgs e)
{
if (lbPerson.SelectedItem == null)
{
return;
}
var pp = (PilePointer)lbPerson.SelectedItem;
m_Pile.Delete(pp);
lbPerson.Items.Remove(pp);
}
public static void NoGrowth_ByteArray(bool speed, int durationSec, int payloadSizeMin, int payloadSizeMax, int countMin, int countMax)
{
using (var pile = new DefaultPile(NOPApplication.Instance))
{
pile.AllocMode = speed ? AllocationMode.FavorSpeed : AllocationMode.ReuseSpace;
pile.Start();
var startTime = DateTime.UtcNow;
var tasks = new List <Task>();
for (var t = 0; t < (System.Environment.ProcessorCount - 1); t++)
{
tasks.Add(Task.Factory.StartNew(() =>
{
var list = new List <CheckByteArray>();
bool put = true;
while (true)
{
if ((DateTime.UtcNow - startTime).TotalSeconds >= durationSec)
{
return;
}
if (put)
{
var cnt = Ambient.Random.NextScaledRandomInteger(countMin, countMax);
for (int j = 0; j < cnt; j++)
{
var payloadSize = Ambient.Random.NextScaledRandomInteger(payloadSizeMin, payloadSizeMax);
var val = new byte[payloadSize];
val[0] = (byte)Ambient.Random.NextRandomInteger;
val[payloadSize - 1] = (byte)Ambient.Random.NextRandomInteger;
var ptr = pile.Put(val);
var element = new CheckByteArray(ptr, payloadSize - 1, val[0], val[payloadSize - 1]);
list.Add(element);
}
Console.WriteLine("Thread {0} put {1} objects".Args(Thread.CurrentThread.ManagedThreadId, list.Count));
put = false;
}
else
{
Console.WriteLine("Thread {0} deleted {1} objects".Args(Thread.CurrentThread.ManagedThreadId, list.Count));
for (var j = 0; j < list.Count; j++)
{
var element = list[j];
var buf = pile.Get(element.Ptr) as byte[];
Aver.AreEqual(element.FirstByte, buf[0]);
Aver.AreEqual(element.LastByte, buf[element.IdxLast]);
pile.Delete(element.Ptr);
}
list.Clear();
put = true;
}
}
}, TaskCreationOptions.LongRunning));
}
Task.WaitAll(tasks.ToArray());
}
Console.WriteLine("Test finished.");
}
public void FitPreallocateByteArray()
{
using (var pile = new DefaultPile())
{
pile.Start();
var obj1 = new byte[3];
var p1 = pile.Put(obj1, preallocateBlockSize: 4000);
var got = pile.Get(p1) as byte[];
Aver.AreEqual(3, got.Length);
Aver.AreEqual(4000, pile.SizeOf(p1));
var obj2 = new byte[571];
Aver.IsTrue(pile.Put(p1, obj2));
Aver.AreEqual(1, pile.ObjectCount);
got = pile.Get(p1) as byte[];
Aver.AreEqual(571, got.Length);
Aver.IsTrue(pile.AllocatedMemoryBytes > 0);
Aver.IsTrue(pile.Delete(p1));
Aver.IsTrue(pile.AllocatedMemoryBytes == 0);
Aver.IsTrue(pile.ObjectCount == 0);
}
}
public void LinkNoPreallocate()
{
using (var pile = new DefaultPile())
{
pile.Start();
var obj1 = new Payload {
ID = 1, Name = "1", Data = null
};
var p1 = pile.Put(obj1);
Aver.IsTrue(pile.SizeOf(p1) < 128);
var obj2 = new Payload {
ID = 2, Name = "2", Data = new byte [128]
};
Aver.IsTrue(pile.Put(p1, obj2));
Aver.AreEqual(1, pile.ObjectCount);
var got = pile.Get(p1) as Payload;
Aver.AreEqual(2, got.ID);
Aver.AreEqual("2", got.Name);
Aver.IsNotNull(got.Data);
Aver.AreEqual(128, got.Data.Length);
Aver.IsTrue(pile.AllocatedMemoryBytes > 0);
Aver.IsTrue(pile.Delete(p1));
Aver.IsTrue(pile.AllocatedMemoryBytes == 0);
Aver.IsTrue(pile.ObjectCount == 0);
}
}
public void LinkByteArrayNoPreallocate()
{
using (var pile = new DefaultPile())
{
pile.Start();
var obj1 = new byte[12];
var p1 = pile.Put(obj1);
var got = pile.Get(p1) as byte[];
Aver.AreEqual(12, got.Length);
var obj2 = new byte[389];
Aver.IsTrue(pile.Put(p1, obj2));
Aver.AreEqual(1, pile.ObjectCount);
got = pile.Get(p1) as byte[];
Aver.AreEqual(389, got.Length);
Aver.IsTrue(pile.AllocatedMemoryBytes > 0);
Aver.IsTrue(pile.Delete(p1));
Aver.IsTrue(pile.AllocatedMemoryBytes == 0);
Aver.IsTrue(pile.ObjectCount == 0);
}
}
public void ReallocateInPlace(int len)
{
using (var pile = new DefaultPile())
{
pile.Start();
var lst = new List <PilePointer>();
for (var i = 0; i < len; i++)
{
var ub = i * 10;
var ptr = pile.Put(new byte[0], preallocateBlockSize: ub);
lst.Add(ptr);
for (var j = 0; j < ub + (ub / 2); j++)
{
Aver.IsTrue(pile.Put(ptr, new byte[j]));
}
}
Aver.AreEqual(lst.Count, pile.ObjectCount);
var n = 0;
foreach (var pp in lst)
{
Console.WriteLine("iteration#{0} of {1}".Args(n, lst.Count));
Aver.IsTrue(pile.Delete(pp));
n++;
}
Aver.AreEqual(0, pile.AllocatedMemoryBytes);
Aver.AreEqual(0, pile.ObjectCount);
Aver.AreEqual(0, pile.ObjectLinkCount);
}
}
public void FitPreallocate()
{
using (var pile = new DefaultPile())
{
pile.Start();
var obj1 = new Payload {
ID = 1, Name = "1", Data = null
};
var p1 = pile.Put(obj1, preallocateBlockSize: 4000);
Aver.AreEqual(4000, pile.SizeOf(p1));
var obj2 = new Payload {
ID = 2, Name = "2", Data = new byte [] { 1, 2, 3, 4, 5, 6, 7, 8 }
};
Aver.IsTrue(pile.Put(p1, obj2));
Aver.AreEqual(1, pile.ObjectCount);
var got = pile.Get(p1) as Payload;
Aver.AreEqual(2, got.ID);
Aver.AreEqual("2", got.Name);
Aver.IsNotNull(got.Data);
Aver.AreEqual(8, got.Data.Length);
Aver.IsTrue(pile.AllocatedMemoryBytes > 0);
Aver.IsTrue(pile.Delete(p1));
Aver.IsTrue(pile.AllocatedMemoryBytes == 0);
Aver.IsTrue(pile.ObjectCount == 0);
}
}
public void FitPreallocateString()
{
using (var pile = new DefaultPile())
{
pile.Start();
var obj1 = "abcdefgh";
var p1 = pile.Put(obj1, preallocateBlockSize: 4000);
var got = pile.Get(p1) as string;
Aver.AreEqual("abcdefgh", got);
Aver.AreEqual(4000, pile.SizeOf(p1));
var obj2 = "abcdefghijklmnopqrst0912345";
Aver.IsTrue(pile.Put(p1, obj2));
Aver.AreEqual(1, pile.ObjectCount);
got = pile.Get(p1) as string;
Aver.AreEqual("abcdefghijklmnopqrst0912345", got);
Aver.IsTrue(pile.AllocatedMemoryBytes > 0);
Aver.IsTrue(pile.Delete(p1));
Aver.IsTrue(pile.AllocatedMemoryBytes == 0);
Aver.IsTrue(pile.ObjectCount == 0);
}
}
public void LinkPreallocate()
{
using (var pile = new DefaultPile(NOPApplication.Instance))
{
pile.Start();
var obj1 = new Payload {
ID = 1, Name = "1", Data = null
};
var p1 = pile.Put(obj1, preallocateBlockSize: 2000);
Aver.AreEqual(2000, pile.SizeOf(p1));
var obj2 = new Payload {
ID = 2, Name = "2", Data = new byte [3000]
};
Aver.IsTrue(pile.Put(p1, obj2));
Aver.AreEqual(1, pile.ObjectCount);
var got = pile.Get(p1) as Payload;
Aver.AreEqual(2, got.ID);
Aver.AreEqual("2", got.Name);
Aver.IsNotNull(got.Data);
Aver.AreEqual(3000, got.Data.Length);
Aver.IsTrue(pile.AllocatedMemoryBytes > 0);
Aver.IsTrue(pile.Delete(p1));
Aver.IsTrue(pile.AllocatedMemoryBytes == 0);
Aver.IsTrue(pile.ObjectCount == 0);
}
}
public void LinkStringNoPreallocate()
{
using (var pile = new DefaultPile(NOPApplication.Instance))
{
pile.Start();
var obj1 = "abcdefgh";
var p1 = pile.Put(obj1);
var got = pile.Get(p1) as string;
Aver.AreEqual("abcdefgh", got);
var obj2 = "abcdefghijklmnopqrst0912345";
Aver.IsTrue(pile.Put(p1, obj2));
Aver.AreEqual(1, pile.ObjectCount);
got = pile.Get(p1) as string;
Aver.AreEqual("abcdefghijklmnopqrst0912345", got);
Aver.IsTrue(pile.AllocatedMemoryBytes > 0);
Aver.IsTrue(pile.Delete(p1));
Aver.IsTrue(pile.AllocatedMemoryBytes == 0);
Aver.IsTrue(pile.ObjectCount == 0);
}
}
public static void NoGrowth_TRow(bool speed, int durationSec, int countMin, int countMax)
{
using (var pile = new DefaultPile())
{
pile.AllocMode = speed ? AllocationMode.FavorSpeed : AllocationMode.ReuseSpace;
pile.Start();
var startTime = DateTime.UtcNow;
var tasks = new List <Task>();
for (var t = 0; t < (System.Environment.ProcessorCount - 1); t++)
{
tasks.Add(Task.Factory.StartNew(() =>
{
var list = new List <CheckTRow>();
bool put = true;
while (true)
{
if ((DateTime.UtcNow - startTime).TotalSeconds >= durationSec)
{
return;
}
if (put)
{
var cnt = NFX.ExternalRandomGenerator.Instance.NextScaledRandomInteger(countMin, countMax);
for (int j = 0; j < cnt; j++)
{
var val = PersonRow.MakeFake(new GDID());
var ptr = pile.Put(val);
var element = new CheckTRow(ptr, val.ID, val.Address1);
list.Add(element);
}
Console.WriteLine("Thread {0} put {1} objects".Args(Thread.CurrentThread.ManagedThreadId, list.Count));
put = false;
}
else
{
Console.WriteLine("Thread {0} deleted {1} objects".Args(Thread.CurrentThread.ManagedThreadId, list.Count));
for (var j = 0; j < list.Count; j++)
{
var element = list[j];
var buf = pile.Get(element.Ptr) as PersonRow;
Aver.AreEqual(element.Id, buf.ID);
Aver.AreEqual(element.Address, buf.Address1);
pile.Delete(element.Ptr);
}
list.Clear();
put = true;
}
}
}, TaskCreationOptions.LongRunning));
}
Task.WaitAll(tasks.ToArray());
}
Console.WriteLine("Test finished.");
}
public void PileDeleteInMiddleSegment(int payloadSize)
{
using (var pile = new DefaultPile()
{
SegmentSize = PileCacheTestCore.SEG_SIZE
})
{
pile.Start();
var pps = new List <PilePointer>();
while (pile.SegmentCount < 4)
{
pps.Add(pile.Put(generatePayload(payloadSize)));
}
pile.Delete(pps.Last());
pps.RemoveAt(pps.Count - 1);
var objectsInsegmentCount = pps.Count / 3;
Console.WriteLine("{0:N0} object in pile, {1:N0} object per segment", pile.ObjectCount, objectsInsegmentCount);
Assert.AreEqual(3, pile.SegmentCount);
Assert.AreEqual(3, pile.SegmentTotalCount);
for (int i = objectsInsegmentCount; i < 2 * objectsInsegmentCount; i++)
{
pile.Delete(pps[i]);
}
Console.WriteLine("{0:N0} object in pile, {1:N0} segments, {2:N0} segments total", pile.ObjectCount, pile.SegmentCount, pile.SegmentTotalCount);
Assert.AreEqual(2, pile.SegmentCount);
Assert.AreEqual(3, pile.SegmentTotalCount);
}
}
public void StringCorrectess(int len, int deleteEvery, int parallel)
{
using (var pile = new DefaultPile())
{
pile.Start();
var bag = new ConcurrentBag <PilePointer>();
var deleted = 0;
Parallel.For(0, len, new ParallelOptions {
MaxDegreeOfParallelism = parallel
},
(i) =>
{
var str = new string('a', i);
PilePointer pp = PilePointer.Invalid;
if (i % 2 == 0)
{
if (bag.TryTake(out pp))
{
pile.Put(pp, str);
}
}
pp = pile.Put(str, preallocateBlockSize: i + 100);
var got = pile.Get(pp) as string;
Aver.AreEqual(str, got);
if (i % deleteEvery == 0)
{
PilePointer dp;
if (bag.TryTake(out dp))
{
if (pile.Delete(dp, false))
{
Interlocked.Increment(ref deleted);
}
}
bag.Add(pp);
}
});
Console.WriteLine("Deleted {0:n0}", deleted);
}
}
public void ByteCorrectess(int len, int deleteEvery, int parallel)
{
using (var pile = new DefaultPile(NOPApplication.Instance))
{
pile.Start();
var bag = new ConcurrentBag <PilePointer>();
var deleted = 0;
Parallel.For(0, len, new ParallelOptions {
MaxDegreeOfParallelism = parallel
},
(i) =>
{
var original = new byte[len - i];
var pp = pile.Put(original);
var got = pile.Get(pp) as byte[];
Aver.AreEqual(original.Length, got.Length);
if (i % deleteEvery == 0)
{
PilePointer dp;
if (bag.TryTake(out dp))
{
if (pile.Delete(dp, false))
{
Interlocked.Increment(ref deleted);
}
}
bag.Add(pp);
}
});
Console.WriteLine("Deleted {0:n0}", deleted);
}
}
public void Reallocate_Delete()
{
using (var pile = new DefaultPile())
{
pile.Start();
var obj1 = new Payload {
ID = 1, Name = "1", Data = null
};
var p1 = pile.Put(obj1, preallocateBlockSize: 2000);
Aver.AreEqual(2000, pile.SizeOf(p1));
var a = pile.AllocatedMemoryBytes;
var u = pile.UtilizedBytes;
var o = pile.OverheadBytes;
Console.WriteLine("Allocated: {0:n0} Utilized: {1:n0} Overhead: {2:n0}", a, u, o);
Aver.AreEqual(2000, u);
Aver.IsTrue(pile.Put(p1, new Payload {
Data = new byte[8000]
}));
var a2 = pile.AllocatedMemoryBytes;
var u2 = pile.UtilizedBytes;
var o2 = pile.OverheadBytes;
Console.WriteLine("Allocated: {0:n0} Utilized: {1:n0} Overhead: {2:n0}", a2, u2, o2);
Aver.IsTrue(u2 > 10000);
Aver.IsTrue(u2 > u);
pile.Delete(p1);
u2 = pile.UtilizedBytes;
Console.WriteLine("Allocated: {0:n0} Utilized: {1:n0} Overhead: {2:n0}", a2, u2, o2);
Aver.AreEqual(0, u2);
}
}
public void PileDeleteInLastSegment(int payloadSize)
{
using (var pile = new DefaultPile()
{
SegmentSize = PileCacheTestCore.SEG_SIZE
})
{
pile.Start();
var pps = new List <PilePointer>();
while (pile.SegmentCount < 2)
{
pps.Add(pile.Put(generatePayload(payloadSize)));
}
pile.Delete(pps.Last());
pps.RemoveAt(pps.Count - 1);
Console.WriteLine("segment count: {0}, segment total count: {1}", pile.SegmentCount, pile.SegmentTotalCount);
Assert.AreEqual(1, pile.SegmentCount);
Assert.AreEqual(1, pile.SegmentTotalCount);
}
}
public static void DeleteSeveral_TRow(bool speed, int durationSec, int putMin, int putMax, int delFactor, bool isParallel)
{
using (var pile = new DefaultPile())
{
pile.AllocMode = speed ? AllocationMode.FavorSpeed : AllocationMode.ReuseSpace;
pile.Start();
var startTime = DateTime.UtcNow;
var tasks = new List <Task>();
for (var t = 0; t < (isParallel ? (System.Environment.ProcessorCount - 1) : 1); t++)
{
tasks.Add(Task.Factory.StartNew(() =>
{
var list = new List <CheckTRow>();
var wlc = 0;
while (true)
{
if ((DateTime.UtcNow - startTime).TotalSeconds >= durationSec)
{
break;
}
var putCount = NFX.ExternalRandomGenerator.Instance.NextScaledRandomInteger(putMin, putMax);
for (int i = 0; i < putCount; i++)
{
var val = PersonRow.MakeFake(new GDID());
var ptr = pile.Put(val);
list.Add(new CheckTRow(ptr, val.ID, val.Address1));
}
// delete several random elements
int delCount = putCount / delFactor;
for (int i = 0; i < delCount; i++)
{
var idx = NFX.ExternalRandomGenerator.Instance.NextScaledRandomInteger(0, list.Count - 1);
var ptr = list[idx].Ptr;
pile.Delete(ptr);
list.RemoveAt(idx);
}
// get several random elements
if (list.Count > 64 && NFX.ExternalRandomGenerator.Instance.NextScaledRandomInteger(0, 100) > 98)
{
var toRead = NFX.ExternalRandomGenerator.Instance.NextScaledRandomInteger(8, 64);
wlc++;
if (wlc % 125 == 0)
{
Console.WriteLine("Thread {0} is reading {1} elements, total {2}"
.Args(Thread.CurrentThread.ManagedThreadId, toRead, list.Count));
}
for (var k = 0; k < toRead; k++)
{
var element = list[NFX.ExternalRandomGenerator.Instance.NextScaledRandomInteger(0, list.Count - 1)];
var buf = pile.Get(element.Ptr) as PersonRow;
Aver.IsTrue(element.Id.Equals(buf.ID));
Aver.IsTrue(element.Address.Equals(buf.Address1));
}
}
}
// total check
Console.WriteLine("Thread {0} is doing final read of {1} elements, objectCount {2}"
.Args(Thread.CurrentThread.ManagedThreadId, list.Count, pile.ObjectCount));
foreach (var element in list)
{
var buf = pile.Get(element.Ptr) as PersonRow;
Aver.IsTrue(element.Id.Equals(buf.ID));
Aver.IsTrue(element.Address.Equals(buf.Address1));
}
return;
}, TaskCreationOptions.LongRunning));
}
Task.WaitAll(tasks.ToArray());
}
}
public void PileSmallObjects(int payloadSize, params int[] freeChunkSizes)
{
using (var pile = new DefaultPile()
{
SegmentSize = PileCacheTestCore.SEG_SIZE, AllocMode = AllocationMode.ReuseSpace
})
{
pile.FreeChunkSizes = freeChunkSizes;
pile.Start();
var pps = new List <PilePointer>();
while (pile.SegmentCount < 2)
{
pps.Add(pile.Put(generatePayload(payloadSize)));
}
pile.Delete(pps.Last());
pps.RemoveAt(pps.Count - 1);
Console.WriteLine("just removed the last added payload and segment should be 1 now, real segment count is {0}", pile.SegmentCount);
var objectsInFirstSegment = pps.Count;
Console.WriteLine("put {0:N0} objects in first segment, pile.ObjectCount {1:N0}", objectsInFirstSegment, pile.ObjectCount);
var deletedObjectCount = 0;
for (int i = 0; i < pps.Count; i += 2, deletedObjectCount++)
{
Assert.IsTrue(pile.Delete(pps[i]));
}
Console.WriteLine("deleted {0:N0} objects, pile.ObjectCount {1:N0}", deletedObjectCount, pile.ObjectCount);
var crawlStatus = pile.Crawl(false);
Console.WriteLine("crawl: {0}", crawlStatus);
var pps1 = new List <PilePointer>();
var c = 0;
while (pile.SegmentCount < 3)
{
pps1.Add(pile.Put(generatePayload(payloadSize)));
if (c % 20000 == 0)
{
pile.Crawl(true); //we do crawl because otherwise the 25000 free index xlots get exhausted AND
}
c++; //this unit tests does not run long enough to cause Crawl within allocator (5+ seconds)
} //so we induce Crawl by hand to rebiild indexes
pile.Delete(pps1.Last());
pps1.RemoveAt(pps1.Count - 1);
Console.WriteLine("again just removed the last added payload and segment should be 2 now, real segment count is {0}", pile.SegmentCount);
var objectsInSecondSegment = pps1.Count;
Console.WriteLine("put {0:N0} objects in second segment, pile.ObjectCount {1:N0}", objectsInSecondSegment, pile.ObjectCount);
Assert.AreEqual(objectsInFirstSegment * 2, pile.ObjectCount, "Object count in pile with two full segments must be equal to double object count in full first segment!");
}
}
public static void PutGetDelete_Parallel(int fromSize, int toSize, int fromObjCount, int toObjCount, int taskCount)
{
Console.WriteLine("test will take about 1 minute");
int objectsPut = 0, objectsDeleted = 0;//, objectGot = 0;
using (var pile = new DefaultPile(NOPApplication.Instance)
{
SegmentSize = SEG_SIZE
})
{
pile.Start();
Parallel.For(0, taskCount, _ =>
{
var startTime = DateTime.Now;
var objects = new Dictionary <PilePointer, byte[]>();
var objectCount = 0;
var objectsSumSize = 0;
var dtStop = DateTime.Now.AddMinutes(.5);
while (dtStop >= DateTime.Now)
{
// insert routine
var insertCount = Ambient.Random.NextScaledRandomInteger(fromObjCount, toObjCount);
for (int i = 0; i < insertCount; i++)
{
var payloadSize = Ambient.Random.NextScaledRandomInteger(fromSize, toSize);
var payload = new byte[payloadSize];
payload[0] = (byte)Ambient.Random.NextScaledRandomInteger(0, 255);
payload[payloadSize - 1] = (byte)Ambient.Random.NextScaledRandomInteger(0, 255);
var pp = pile.Put(payload);
objects.Add(pp, payload);
objectCount++;
objectsSumSize += payloadSize;
}
Interlocked.Add(ref objectsPut, insertCount);
// get
if (objectCount > 0)
{
var getCount = Ambient.Random.NextScaledRandomInteger(5 * fromObjCount, 5 * toObjCount);
for (int i = 0; i < getCount; i++)
{
var objectIdx = Ambient.Random.NextScaledRandomInteger(0, objectCount - 1);
var obj = objects.ElementAt(objectIdx);
var objPayloadFromPile = (byte[])pile.Get(obj.Key);
Aver.AreEqual(obj.Value[0], objPayloadFromPile[0]);
Aver.AreEqual(obj.Value[obj.Value.Length - 1], objPayloadFromPile[obj.Value.Length - 1]);
Aver.AreEqual(obj.Value.Length, objPayloadFromPile.Length);
}
}
// delete
var deleteCount = Ambient.Random.NextScaledRandomInteger(fromObjCount, toObjCount);
if (deleteCount > objectCount)
{
deleteCount = objectCount;
}
for (int i = 0; i < deleteCount; i++)
{
if (objectCount == 0)
{
break;
}
var objectIdx = Ambient.Random.NextScaledRandomInteger(0, objectCount - 1);
var obj = objects.ElementAt(objectIdx);
Aver.IsTrue(pile.Delete(obj.Key));
objects.Remove(obj.Key);
objectCount--; objectsSumSize -= obj.Value.Length;
}
Interlocked.Add(ref objectsDeleted, deleteCount);
}
});
}
Console.WriteLine("put {0:N0}, deleted {1:N0} object", objectsPut, objectsDeleted);
}
public static void Chessboard_ByteArray(bool speed, int durationSec, int payloadSizeMin, int payloadSizeMax, bool isParallel)
{
using (var pile = new DefaultPile())
{
pile.AllocMode = speed ? AllocationMode.FavorSpeed : AllocationMode.ReuseSpace;
pile.Start();
var startTime = DateTime.UtcNow;
var tasks = new List<Task>();
for (var t = 0; t < (isParallel ? (System.Environment.ProcessorCount - 1) : 1); t++)
tasks.Add(Task.Factory.StartNew(() =>
{
var list = new List<CheckByteArray>();
var i = 0;
var wlc = 0;
while (true)
{
if ((DateTime.UtcNow - startTime).TotalSeconds >= durationSec) break;
var payloadSize = NFX.ExternalRandomGenerator
.Instance.NextScaledRandomInteger(payloadSizeMin, payloadSizeMax);
var val = new byte[payloadSize];
val[0] = (byte)NFX.ExternalRandomGenerator.Instance.NextRandomInteger;
val[payloadSize - 1] = (byte)NFX.ExternalRandomGenerator.Instance.NextRandomInteger;
var ptr = pile.Put(val);
var element = new CheckByteArray(ptr, payloadSize - 1, val[0], val[payloadSize - 1]);
list.Add(element);
// delete previous element
if (list.Count > 1 && i % 2 == 0)
{
ptr = list[list.Count - 2].Ptr;
pile.Delete(ptr);
list.RemoveAt(list.Count - 2);
}
// get several random elements
if (list.Count > 64 && NFX.ExternalRandomGenerator.Instance.NextScaledRandomInteger(0, 100) > 98)
{
var toRead = NFX.ExternalRandomGenerator.Instance.NextScaledRandomInteger(8, 64);
wlc++;
if (wlc % 125 == 0)
Console.WriteLine("Thread {0} is reading {1} elements, total {2}, Pile objects {3}, Pile segments {4} Pile Bytes {5}"
.Args(Thread.CurrentThread.ManagedThreadId, toRead, list.Count, pile.ObjectCount, pile.SegmentCount, pile.AllocatedMemoryBytes));
for (var k = 0; k < toRead; k++)
{
element = list[NFX.ExternalRandomGenerator.Instance.NextScaledRandomInteger(0, list.Count - 1)];
var buf = pile.Get(element.Ptr) as byte[];
Assert.AreEqual(element.FirstByte, buf[0]);
Assert.AreEqual(element.LastByte, buf[element.IdxLast]);
}
}
if (i == Int32.MaxValue)
i = 0;
else
i++;
if (list.Count == Int32.MaxValue)
list = new List<CheckByteArray>();
}
// total check
Console.WriteLine("Thread {0} is doing final read of {1} elements, ObjectCount {2}"
.Args(Thread.CurrentThread.ManagedThreadId, list.Count, pile.ObjectCount));
foreach (var element in list)
{
var buf = pile.Get(element.Ptr) as byte[];
Assert.AreEqual(element.FirstByte, buf[0]);
Assert.AreEqual(element.LastByte, buf[element.IdxLast]);
}
return;
}, TaskCreationOptions.LongRunning));
Task.WaitAll(tasks.ToArray());
}
}
public void TestThreadSafety(int cnt, int tcount, int seconds)
{
using (var pile = new DefaultPile())
{
// pile.SegmentSize = 64 * 1024 * 1024;
pile.Start();
var data = new PilePointer[cnt];
for (var i = 0; i < cnt; i++)
{
data[i] = pile.Put(new byte[0], preallocateBlockSize: Memory.PTR_RAW_BYTE_SIZE + (137 * (i % 17)));
}
var lst = new List <Task>();
long statRead = 0;
long statPut = 0;
long statDelete = 0;
var sw = Stopwatch.StartNew();
var sd = DateTime.UtcNow;
var deleteLock = new object();
for (var i = 0; i < tcount; i++)
{
lst.Add(Task.Factory.StartNew(() =>
{
while (true)
{
var now = DateTime.UtcNow;
if ((now - sd).TotalSeconds > seconds)
{
break;
}
var it = ExternalRandomGenerator.Instance.NextScaledRandomInteger(10, 100);
for (var j = 0; j < it; j++)
{
var pp = data[ExternalRandomGenerator.Instance.NextScaledRandomInteger(0, cnt - 1)];
Aver.IsTrue(pile.Get(pp) is byte[]);
Interlocked.Increment(ref statRead);
}
it = ExternalRandomGenerator.Instance.NextScaledRandomInteger(10, 100);
for (var j = 0; j < it; j++)
{
var pp = data[ExternalRandomGenerator.Instance.NextScaledRandomInteger(0, cnt - 1)];
Aver.IsTrue(pile.Put(pp, new byte[ExternalRandomGenerator.Instance.NextScaledRandomInteger(0, 3791)], link: true));
Interlocked.Increment(ref statPut);
}
if (ExternalRandomGenerator.Instance.NextScaledRandomInteger(0, 100) > 50 && Monitor.TryEnter(deleteLock))
{
try
{
var newData = (PilePointer[])data.Clone();
it = ExternalRandomGenerator.Instance.NextScaledRandomInteger(0, 10 + (cnt / 2));
var toDelete = new List <PilePointer>();
for (var j = 0; j < it; j++)
{
var idx = ExternalRandomGenerator.Instance.NextScaledRandomInteger(0, cnt - 1);
toDelete.Add(newData[idx]);
newData[idx] = pile.Put(new byte[12], preallocateBlockSize: ExternalRandomGenerator.Instance.NextScaledRandomInteger(24, 1024));
}
data = newData;//atomic;
Thread.Sleep(1000);
foreach (var pp in toDelete)
{
Aver.IsTrue(pile.Delete(pp));
Interlocked.Increment(ref statDelete);
}
}
finally
{
Monitor.Exit(deleteLock);
}
}
}
}, TaskCreationOptions.LongRunning));
}
Task.WaitAll(lst.ToArray());
var el = sw.ElapsedMilliseconds;
Console.WriteLine("Read {0:n0} at {1:n0} ops/sec".Args(statRead, statRead / (el / 1000d)));
Console.WriteLine("Put {0:n0} at {1:n0} ops/sec".Args(statPut, statPut / (el / 1000d)));
Console.WriteLine("Deleted {0:n0} at {1:n0} ops/sec".Args(statDelete, statDelete / (el / 1000d)));
for (var i = 0; i < data.Length; i++)
{
Aver.IsTrue(pile.Delete(data[i]));
}
Aver.AreEqual(0, pile.ObjectCount);
Aver.AreEqual(0, pile.ObjectLinkCount);
Aver.AreEqual(0, pile.AllocatedMemoryBytes);
}
}
public static void PutGetDelete_Sequential(int fromSize, int toSize, int fromObjCount, int toObjCount)
{
Console.WriteLine("test will take about 1 minute");
var startTime = DateTime.Now;
var objects = new Dictionary <PilePointer, byte[]>();
var objectCount = 0;
var objectsSumSize = 0;
using (var pile = new DefaultPile()
{
SegmentSize = PileCacheTestCore.SEG_SIZE
})
{
pile.Start();
var dtStop = DateTime.Now.AddMinutes(1);
while (dtStop >= DateTime.Now)
{
// insert routine
var insertCount = NFX.ExternalRandomGenerator.Instance.NextScaledRandomInteger(fromObjCount, toObjCount);
for (int i = 0; i < insertCount; i++)
{
var payloadSize = NFX.ExternalRandomGenerator.Instance.NextScaledRandomInteger(fromSize, toSize);
var payload = new byte[payloadSize];
payload[0] = (byte)NFX.ExternalRandomGenerator.Instance.NextScaledRandomInteger(0, 255);
payload[payloadSize - 1] = (byte)NFX.ExternalRandomGenerator.Instance.NextScaledRandomInteger(0, 255);
var pp = pile.Put(payload);
objects.Add(pp, payload);
objectCount++;
objectsSumSize += payloadSize;
}
// get
if (objectCount > 0)
{
var getCount = ExternalRandomGenerator.Instance.NextScaledRandomInteger(5 * fromObjCount, 5 * toObjCount);
for (int i = 0; i < getCount; i++)
{
var objectIdx = ExternalRandomGenerator.Instance.NextScaledRandomInteger(0, objectCount - 1);
var obj = objects.ElementAt(objectIdx);
var objPayloadFromPile = (byte[])pile.Get(obj.Key);
Assert.AreEqual(obj.Value[0], objPayloadFromPile[0]);
Assert.AreEqual(obj.Value[obj.Value.Length - 1], objPayloadFromPile[obj.Value.Length - 1]);
Assert.AreEqual(obj.Value.Length, objPayloadFromPile.Length);
}
}
// delete
var deleteCount = ExternalRandomGenerator.Instance.NextScaledRandomInteger(fromObjCount, toObjCount);
for (int i = 0; i < deleteCount; i++)
{
if (objectCount == 0)
{
break;
}
var objectIdx = ExternalRandomGenerator.Instance.NextScaledRandomInteger(0, objectCount - 1);
var obj = objects.ElementAt(objectIdx);
Assert.IsTrue(pile.Delete(obj.Key));
objects.Remove(obj.Key);
objectCount--; objectsSumSize -= obj.Value.Length;
}
}
}
}
public void VarSizes_Checkboard(bool isParallel, int cnt, int minSz, int maxSz, bool speed)
{
using (var pile = new DefaultPile())
{
pile.Start();
pile.AllocMode = speed ? AllocationMode.FavorSpeed : AllocationMode.ReuseSpace;
var tasks = new List <Task>();
for (var t = 0; t < (isParallel? (System.Environment.ProcessorCount - 1) : 1); t++)
{
tasks.Add(
Task.Run(() =>
{
var dict = new Dictionary <PilePointer, dummy>();
var priorOdd = PilePointer.Invalid;
for (var i = 0; i < cnt; i++)
{
var even = (i & 0x01) == 0;
var data = new dummy {
bin = new byte[12 + NFX.ExternalRandomGenerator.Instance.NextScaledRandomInteger(minSz, maxSz)]
};
data.bin.WriteBEInt32(0, ExternalRandomGenerator.Instance.NextRandomInteger);
data.bin.WriteBEInt32(data.bin.Length - 4, ExternalRandomGenerator.Instance.NextRandomInteger);
var ptr = pile.Put(data);
Assert.IsTrue(ptr.Valid);
if (even)
{
dict.Add(ptr, data);
}
else
{
if (priorOdd.Valid)
{
Assert.IsTrue(pile.Delete(priorOdd));
}
priorOdd = ptr;
}
if (i % 1000 == 0)
{
Console.WriteLine("Thread{0} did {1}; allocated {2} bytes, utilized {3} bytes by {4} objects {5} bytes/obj. ",
Thread.CurrentThread.ManagedThreadId,
i,
pile.AllocatedMemoryBytes,
pile.UtilizedBytes,
pile.ObjectCount,
pile.UtilizedBytes / pile.ObjectCount);
}
}
Console.WriteLine("Thread {0} Population done, now checking the buffers... {1}", Thread.CurrentThread.ManagedThreadId, DateTime.Now);
foreach (var entry in dict)
{
Assert.IsTrue(NFX.IOMiscUtils.MemBufferEquals(entry.Value.bin, (pile.Get(entry.Key) as dummy).bin));
}
Console.WriteLine("Thread {0} DONE. {1}", Thread.CurrentThread.ManagedThreadId, DateTime.Now);
})
);//add
}
Task.WaitAll(tasks.ToArray());
}
}
public void VarSizes_Increasing_Random(bool isParallel, int cnt, int minSz, int maxSz, bool speed, bool rnd)
{
using (var pile = new DefaultPile())
{
pile.Start();
pile.AllocMode = speed ? AllocationMode.FavorSpeed : AllocationMode.ReuseSpace;
var sw = Stopwatch.StartNew();
var tasks = new List <Task>();
for (var t = 0; t < (isParallel? (System.Environment.ProcessorCount - 1) : 1); t++)
{
tasks.Add(
Task.Run(() =>
{
var dict = new Dictionary <PilePointer, dummy>();
var lst = new List <PilePointer>();
var priorOdd = PilePointer.Invalid;
for (var i = 0; i < cnt; i++)
{
var buf = new byte[12 +
minSz +
(
rnd
? (NFX.ExternalRandomGenerator.Instance.NextScaledRandomInteger(0, (int)(maxSz * (i / (double)cnt))))
: (int)(maxSz * (i / (double)cnt))
)
];
var data = new dummy {
bin = buf
};
data.bin.WriteBEInt32(0, ExternalRandomGenerator.Instance.NextRandomInteger);
data.bin.WriteBEInt32(data.bin.Length - 4, ExternalRandomGenerator.Instance.NextRandomInteger);
var ptr = pile.Put(data);
Assert.IsTrue(ptr.Valid);
dict.Add(ptr, data);
lst.Add(ptr);
if (i > cnt / 3)
{
if (ExternalRandomGenerator.Instance.NextRandomInteger > 0)
{
var ri = ExternalRandomGenerator.Instance.NextScaledRandomInteger(0, lst.Count - 1);
var pp = lst[ri];
if (!pp.Valid)
{
continue;
}
Assert.IsTrue(pile.Delete(pp));
dict.Remove(pp);
lst[ri] = PilePointer.Invalid;
}
}
if (i % 1000 == 0)
{
Console.WriteLine("Thread{0} did {1}; allocated {2} bytes, utilized {3} bytes by {4} objects {5} bytes/obj. ",
Thread.CurrentThread.ManagedThreadId,
i,
pile.AllocatedMemoryBytes,
pile.UtilizedBytes,
pile.ObjectCount,
pile.UtilizedBytes / pile.ObjectCount);
}
}
Console.WriteLine("Thread {0} Population done, now checking the buffers... {1}", Thread.CurrentThread.ManagedThreadId, DateTime.Now);
foreach (var entry in dict)
{
Assert.IsTrue(NFX.IOMiscUtils.MemBufferEquals(entry.Value.bin, (pile.Get(entry.Key) as dummy).bin));
}
Console.WriteLine("Thread {0} DONE. {1}", Thread.CurrentThread.ManagedThreadId, DateTime.Now);
})
);//add
}
Task.WaitAll(tasks.ToArray());
var el = sw.ElapsedMilliseconds;
var gt = cnt * tasks.Count;
Console.WriteLine("Total objects: {0:n0} in {1:n0} ms at {2:n0} obj/sec".Args(gt, el, gt / (el / 1000d)));
}
}
public static void Chessboard_TRow(bool speed, int durationSec, bool isParallel)
{
using (var pile = new DefaultPile(NOPApplication.Instance))
{
pile.AllocMode = speed ? AllocationMode.FavorSpeed : AllocationMode.ReuseSpace;
pile.Start();
var startTime = DateTime.UtcNow;
var tasks = new List <Task>();
for (var t = 0; t < (isParallel ? (System.Environment.ProcessorCount - 1) : 1); t++)
{
tasks.Add(Task.Factory.StartNew(() =>
{
var list = new List <CheckTRow>();
var i = 0;
var wlc = 0;
while (true)
{
if ((DateTime.UtcNow - startTime).TotalSeconds >= durationSec)
{
break;
}
var val = PersonRow.MakeFake(new GDID(0, (ulong)i));
var ptr = pile.Put(val);
var element = new CheckTRow(ptr, val.ID, val.Address1);
list.Add(element);
// delete previous element
if (list.Count > 1 && i % 2 == 0)
{
ptr = list[list.Count - 2].Ptr;
pile.Delete(ptr);
list.RemoveAt(list.Count - 2);
}
// get several random elements
if (list.Count > 64 && Ambient.Random.NextScaledRandomInteger(0, 100) > 98)
{
var toRead = Ambient.Random.NextScaledRandomInteger(8, 64);
wlc++;
if (wlc % 125 == 0)
{
Console.WriteLine("Thread {0} is reading {1} elements, total {2}, Pile objects {3}, Pile segments {4} Pile Bytes {5}"
.Args(Thread.CurrentThread.ManagedThreadId, toRead, list.Count, pile.ObjectCount, pile.SegmentCount, pile.AllocatedMemoryBytes));
}
for (var k = 0; k < toRead; k++)
{
element = list[Ambient.Random.NextScaledRandomInteger(0, list.Count - 1)];
var buf = pile.Get(element.Ptr) as PersonRow;
Aver.IsTrue(element.Id.Equals(buf.ID));
Aver.IsTrue(element.Address.Equals(buf.Address1));
}
}
if (i == Int32.MaxValue)
{
i = 0;
}
else
{
i++;
}
if (list.Count == Int32.MaxValue)
{
list = new List <CheckTRow>();
}
}
// total check
Console.WriteLine("Thread {0} is doing final read of {1} elements, objectCount {2}"
.Args(Thread.CurrentThread.ManagedThreadId, list.Count, pile.ObjectCount));
foreach (var element in list)
{
var buf = pile.Get(element.Ptr) as PersonRow;
Aver.IsTrue(element.Id.Equals(buf.ID));
Aver.IsTrue(element.Address.Equals(buf.Address1));
}
return;
}, TaskCreationOptions.LongRunning));
}
Task.WaitAll(tasks.ToArray());
}
}
public static void Put_RandomDelete_ByteArray(int cnt, int durationSec, bool speed, int payloadSizeMin, int payloadSizeMax, int deleteFreq, bool isParallel)
{
using (var pile = new DefaultPile())
{
pile.AllocMode = speed ? AllocationMode.FavorSpeed : AllocationMode.ReuseSpace;
pile.Start();
var startTime = DateTime.UtcNow;
var tasks = new List <Task>();
for (var t = 0; t < (isParallel ? (System.Environment.ProcessorCount - 1) : 1); t++)
{
tasks.Add(Task.Factory.StartNew(() =>
{
var wlc = 0;
while (true)
{
if ((DateTime.UtcNow - startTime).TotalSeconds >= durationSec)
{
return;
}
var dict = new Dictionary <int, CheckByteArray>();
Console.WriteLine("Starting a batch of {0}".Args(cnt));
for (int i = 0; i < cnt; i++)
{
var payloadSize = NFX.ExternalRandomGenerator.Instance.NextScaledRandomInteger(payloadSizeMin, payloadSizeMax);
var val = new byte[payloadSize];
val[0] = (byte)NFX.ExternalRandomGenerator.Instance.NextRandomInteger;
val[payloadSize - 1] = (byte)NFX.ExternalRandomGenerator.Instance.NextRandomInteger;
var ptr = pile.Put(val);
var element = new CheckByteArray(ptr, payloadSize - 1, val[0], val[payloadSize - 1]);
dict.Add(i, element);
if (dict.Count > 0 && i % deleteFreq == 0)
{
while (true)
{
var idx = i - NFX.ExternalRandomGenerator.Instance.NextScaledRandomInteger(0, i);
CheckByteArray stored;
if (dict.TryGetValue(idx, out stored))
{
ptr = stored.Ptr;
pile.Delete(ptr);
dict.Remove(idx);
break;
}
}
}
if (dict.Count > 16 && NFX.ExternalRandomGenerator.Instance.NextScaledRandomInteger(0, 100) > 98)
{
var toRead = NFX.ExternalRandomGenerator.Instance.NextScaledRandomInteger(8, 64);
wlc++;
if (wlc % 125 == 0)
{
Console.WriteLine("Thread {0} is reading {1} elements, total {2}"
.Args(Thread.CurrentThread.ManagedThreadId, toRead, dict.Count));
}
for (var k = 0; k < toRead; k++)
{
var kvp = dict.Skip(NFX.ExternalRandomGenerator.Instance.NextScaledRandomInteger(0, dict.Count - 1)).First();
var buf = pile.Get(kvp.Value.Ptr) as byte[];
Aver.AreEqual(kvp.Value.FirstByte, buf[0]);
Aver.AreEqual(kvp.Value.LastByte, buf[kvp.Value.IdxLast]);
}
}
}
Console.WriteLine("Thread {0} is doing final read of {1} elements".Args(Thread.CurrentThread.ManagedThreadId, dict.Count));
foreach (var kvp in dict)
{
var buf = pile.Get(kvp.Value.Ptr) as byte[];
Aver.AreEqual(kvp.Value.FirstByte, buf[0]);
Aver.AreEqual(kvp.Value.LastByte, buf[kvp.Value.IdxLast]);
}
}
}, TaskCreationOptions.LongRunning));
}
Task.WaitAll(tasks.ToArray());
}
}
public static void DeleteSeveral_ByteArray(bool speed, int durationSec, int putMin, int putMax, int delFactor, int payloadSizeMin, int payloadSizeMax, bool isParallel)
{
using (var pile = new DefaultPile())
{
pile.AllocMode = speed ? AllocationMode.FavorSpeed : AllocationMode.ReuseSpace;
pile.Start();
var startTime = DateTime.UtcNow;
var tasks = new List <Task>();
for (var t = 0; t < (isParallel ? (System.Environment.ProcessorCount - 1) : 1); t++)
{
tasks.Add(Task.Factory.StartNew(() =>
{
var list = new List <CheckByteArray>();
var wlc = 0;
while (true)
{
if ((DateTime.UtcNow - startTime).TotalSeconds >= durationSec)
{
break;
}
var putCount = NFX.ExternalRandomGenerator.Instance.NextScaledRandomInteger(putMin, putMax);
for (int i = 0; i < putCount; i++)
{
var payloadSize = NFX.ExternalRandomGenerator
.Instance.NextScaledRandomInteger(payloadSizeMin, payloadSizeMax);
var val = new byte[payloadSize];
val[0] = (byte)NFX.ExternalRandomGenerator.Instance.NextRandomInteger;
val[payloadSize - 1] = (byte)NFX.ExternalRandomGenerator.Instance.NextRandomInteger;
var ptr = pile.Put(val);
list.Add(new CheckByteArray(ptr, payloadSize - 1, val[0], val[payloadSize - 1]));
}
int delCount = putCount / delFactor;
for (int i = 0; i < delCount; i++)
{
var idx = NFX.ExternalRandomGenerator.Instance.NextScaledRandomInteger(0, list.Count - 1);
var ptr = list[idx].Ptr;
pile.Delete(ptr);
list.RemoveAt(idx);
}
// get several random elements
if (list.Count > 64 && NFX.ExternalRandomGenerator.Instance.NextScaledRandomInteger(0, 100) > 98)
{
var toRead = NFX.ExternalRandomGenerator.Instance.NextScaledRandomInteger(8, 64);
wlc++;
if (wlc % 125 == 0)
{
Console.WriteLine("Thread {0} is reading {1} elements, total {2}"
.Args(Thread.CurrentThread.ManagedThreadId, toRead, list.Count));
}
for (var k = 0; k < toRead; k++)
{
var element = list[NFX.ExternalRandomGenerator.Instance.NextScaledRandomInteger(0, list.Count - 1)];
var buf = pile.Get(element.Ptr) as byte[];
Aver.AreEqual(element.FirstByte, buf[0]);
Aver.AreEqual(element.LastByte, buf[element.IdxLast]);
}
}
if (list.Count == Int32.MaxValue)
{
list = new List <CheckByteArray>();
}
}
// total check
Console.WriteLine("Thread {0} is doing final read of {1} elements, objectCount {2}"
.Args(Thread.CurrentThread.ManagedThreadId, list.Count, pile.ObjectCount));
foreach (var element in list)
{
var buf = pile.Get(element.Ptr) as byte[];
Aver.AreEqual(element.FirstByte, buf[0]);
Aver.AreEqual(element.LastByte, buf[element.IdxLast]);
}
return;
}));
}
Task.WaitAll(tasks.ToArray());
}
}