public void Parallel_PutDeleteGet_Checkerboard()
{
const int CNT = 1002030;//1000203;
var tuples = new Tuple<PilePointer, Charge>[CNT];
using (var pile = new DefaultPile())
{
pile.Start();
var ipile = pile as IPile;
Parallel.For(0, CNT, i => {
var ch = Charge.MakeFake(new GDID(0, (ulong)i));
var pp = ipile.Put(ch);
tuples[i] = new Tuple<PilePointer,Charge>(pp, ch);
});
Assert.AreEqual(CNT, ipile.ObjectCount);
Parallel.For(0, CNT, i => {
if (i % 3 != 0)
ipile.Delete(tuples[i].Item1);
});
Assert.AreEqual(CNT/3, ipile.ObjectCount);
var deletedHits = new ConcurrentDictionary<int, int>();
Parallel.For(0, CNT, i => {
if (i % 3 != 0)
{
try
{
deletedHits.AddOrUpdate(System.Threading.Thread.CurrentThread.ManagedThreadId, 1, (threadId, val) => val + 1);
ipile.Get(tuples[i].Item1);
Assert.Fail("Object is deleted but its pointer doesn't throw exception!");
}
catch (PileAccessViolationException) {}
}
else
{
var ch = ipile.Get(tuples[i].Item1);
Assert.AreEqual(ch, tuples[i].Item2);
}
});
foreach (var kvp in deletedHits)
{
Console.WriteLine("Thread '{0}' {1:n0} times accessed deleted pointer", kvp.Key, kvp.Value);
}
}
}
public void SegmentAddedDeleted()
{
const int SEG_SIZE = DefaultPile.SEG_SIZE_MIN;
using (var pile = new DefaultPile() { SegmentSize = SEG_SIZE})
{
pile.Start();
var ipile = pile as IPile;
var pp = PilePointer.Invalid;
for(ulong i=0; i<SEG_SIZE && ipile.SegmentCount < 2; i++)
{
var ch = Charge.MakeFake(new GDID(1, i));
pp = ipile.Put(ch);
}
Assert.AreEqual(2, ipile.SegmentCount);
ipile.Delete(pp);
Assert.AreEqual(1, ipile.SegmentCount);
}
}
public void Parallel_PutGet()
{
const int CNT = 1000000;
var tuples = new Tuple<PilePointer, Charge>[CNT];
using (var pile = new DefaultPile())
{
pile.Start();
var ipile = pile as IPile;
Parallel.For(0, CNT, i => {
var ch = Charge.MakeFake(new GDID(0, (ulong)i));
var pp = ipile.Put(ch);
tuples[i] = new Tuple<PilePointer,Charge>(pp, ch);
});
Assert.AreEqual(CNT, ipile.ObjectCount);
Parallel.ForEach(tuples, t => {
Assert.AreEqual(t.Item2, ipile.Get(t.Item1));
});
}
}
public void PutCheckerboardPattern3()
{
using (var pile = new DefaultPile())
{
pile.Start();
var ipile = pile as IPile;
const ulong CNT = 123;
var ppp = new Tuple<PilePointer, string>[CNT];
for (ulong i = 0; i < CNT; i++)
{
var str = NFX.Parsing.NaturalTextGenerator.Generate(179);
ppp[i] = new Tuple<PilePointer,string>( ipile.Put(str), str);
}
Assert.AreEqual(CNT, ipile.ObjectCount);
for(ulong i = 0; i < CNT; i++)
{
if (i % 3 != 0)
ipile.Delete(ppp[i].Item1);
}
Assert.AreEqual(CNT/3, ipile.ObjectCount);
for(ulong i = 0; i < CNT; i++)
{
if (i % 3 != 0)
{
try
{
ipile.Get(ppp[i].Item1);
Assert.Fail("Object is deleted but its pointer doesn't throw exception!");
}
catch (PileAccessViolationException) {}
}
else
{
var ch = ipile.Get(ppp[i].Item1);
Assert.AreEqual(ch, ppp[i].Item2);
}
}
////Console.WriteLine("ObjectCount: {0}", ipile.ObjectCount);
////Console.WriteLine("AllocatedMemoryBytes: {0}", ipile.AllocatedMemoryBytes);
////Console.WriteLine("UtilizedBytes: {0}", ipile.UtilizedBytes);
////Console.WriteLine("OverheadBytes: {0}", ipile.OverheadBytes);
////Console.WriteLine("SegmentCount: {0}", ipile.SegmentCount);
}
}
public void PutCheckerboardPattern2()
{
using (var pile = new DefaultPile())
{
pile.Start();
var ipile = pile as IPile;
const ulong CNT = 100;
var ppp = new Tuple<PilePointer, Charge>[CNT];
for (ulong i = 0; i < CNT; i++)
{
var ch = Charge.MakeFake(new GDID(0, i));
ppp[i] = new Tuple<PilePointer,Charge>( ipile.Put(ch), ch);
}
Assert.AreEqual(CNT, ipile.ObjectCount);
for(ulong i = 0; i < CNT; i++)
{
var ch = ipile.Get(ppp[i].Item1);
Assert.AreEqual(ch, ppp[i].Item2);
}
for(ulong i = 0; i < CNT; i+=2)
ipile.Delete(ppp[i].Item1);
Assert.AreEqual(CNT/2, ipile.ObjectCount);
for(ulong i = 0; i < CNT; i++)
{
if (i % 2 == 0)
{
try
{
ipile.Get(ppp[i].Item1);
Assert.Fail("Object is deleted but its pointer doesn't throw exception!");
}
catch (PileAccessViolationException) {}
}
else
{
var ch = ipile.Get(ppp[i].Item1);
Assert.AreEqual(ch, ppp[i].Item2);
}
}
}
}
public static void DeleteSeveral_TRow(bool speed, int durationSec, int putMin, int putMax, int delFactor, bool isParallel)
{
using (var pile = new DefaultPile())
{
pile.AllocMode = speed ? AllocationMode.FavorSpeed : AllocationMode.ReuseSpace;
pile.Start();
var startTime = DateTime.UtcNow;
var tasks = new List<Task>();
for (var t = 0; t < (isParallel ? (System.Environment.ProcessorCount - 1) : 1); t++)
tasks.Add(Task.Factory.StartNew(() =>
{
var list = new List<CheckTRow>();
var wlc = 0;
while (true)
{
if ((DateTime.UtcNow - startTime).TotalSeconds >= durationSec) break;
var putCount = NFX.ExternalRandomGenerator.Instance.NextScaledRandomInteger(putMin, putMax);
for (int i = 0; i < putCount; i++)
{
var val = PersonRow.MakeFake(new GDID());
var ptr = pile.Put(val);
list.Add(new CheckTRow(ptr, val.ID, val.Address1));
}
// delete several random elements
int delCount = putCount / delFactor;
for (int i = 0; i < delCount; i++)
{
var idx = NFX.ExternalRandomGenerator.Instance.NextScaledRandomInteger(0, list.Count - 1);
var ptr = list[idx].Ptr;
pile.Delete(ptr);
list.RemoveAt(idx);
}
// get several random elements
if (list.Count > 64 && NFX.ExternalRandomGenerator.Instance.NextScaledRandomInteger(0, 100) > 98)
{
var toRead = NFX.ExternalRandomGenerator.Instance.NextScaledRandomInteger(8, 64);
wlc++;
if (wlc % 125 == 0)
Console.WriteLine("Thread {0} is reading {1} elements, total {2}"
.Args(Thread.CurrentThread.ManagedThreadId, toRead, list.Count));
for (var k = 0; k < toRead; k++)
{
var element = list[NFX.ExternalRandomGenerator.Instance.NextScaledRandomInteger(0, list.Count - 1)];
var buf = pile.Get(element.Ptr) as PersonRow;
Assert.IsTrue(element.Id.Equals(buf.ID));
Assert.IsTrue(element.Address.Equals(buf.Address1));
}
}
}
// total check
Console.WriteLine("Thread {0} is doing final read of {1} elements, objectCount {2}"
.Args(Thread.CurrentThread.ManagedThreadId, list.Count, pile.ObjectCount));
foreach (var element in list)
{
var buf = pile.Get(element.Ptr) as PersonRow;
Assert.IsTrue(element.Id.Equals(buf.ID));
Assert.IsTrue(element.Address.Equals(buf.Address1));
}
return;
}, TaskCreationOptions.LongRunning));
Task.WaitAll(tasks.ToArray());
}
}
public void T060_PileNonOwnership()
{
var pile = new DefaultPile();
pile.Start();
try
{
var cache = new LocalCache();
cache.Pile = pile;
cache.Configure(null);
cache.Start();
var tA = cache.GetOrCreateTable<string>("A");
tA.Put("aaa", "avalue");
tA.Put("bbb", "bvalue");
Assert.AreEqual("avalue", tA.Get("aaa"));
Assert.AreEqual(2, cache.Count);
Assert.AreEqual(2, pile.ObjectCount);
cache.WaitForCompleteStop();
Assert.AreEqual(NFX.ServiceModel.ControlStatus.Inactive, cache.Status);
Assert.AreEqual(NFX.ServiceModel.ControlStatus.Active, pile.Status);
Assert.AreEqual(0, pile.ObjectCount);
cache = new LocalCache();
cache.Pile = pile;
cache.Configure(null);
cache.Start();
var tAbc = cache.GetOrCreateTable<string>("Abc");
tAbc.Put("aaa", "avalue");
tAbc.Put("bbb", "bvalue");
tAbc.Put("ccc", "cvalue");
tAbc.Put("ddd", "cvalue");
Assert.AreEqual(4, pile.ObjectCount);
var cache2 = new LocalCache();
cache2.Pile = pile;
cache2.Configure(null);
cache2.Start();
var t2 = cache2.GetOrCreateTable<string>("A");
t2.Put("aaa", "avalue");
t2.Put("bbb", "bvalue");
Assert.AreEqual(2, cache2.Count);
Assert.AreEqual(6, pile.ObjectCount);
cache.WaitForCompleteStop();
Assert.AreEqual(NFX.ServiceModel.ControlStatus.Active, pile.Status);
Assert.AreEqual(2, pile.ObjectCount);
cache2.WaitForCompleteStop();
Assert.AreEqual(NFX.ServiceModel.ControlStatus.Active, pile.Status);
Assert.AreEqual(0, pile.ObjectCount);
pile.WaitForCompleteStop();
Assert.AreEqual(NFX.ServiceModel.ControlStatus.Inactive, pile.Status);
}
finally
{
pile.Dispose();
}
}
public void Configuration()
{
var conf = @"
app
{
memory-management
{
pile
{
alloc-mode=favorspeed
free-list-size=100000
max-segment-limit=79
segment-size=395313143 //will be rounded to 16 byte boundary: 395,313,152
max-memory-limit=123666333000
free-chunk-sizes='128, 256, 512, 1024, 2000, 3000, 4000, 5000, 6000, 7000, 8000, 9000, 16000, 32000, 64000, 256000'
}
pile
{
name='specialNamed'
free-list-size=99000
max-segment-limit=73
segment-size=395313147 //will be rounded to 16 byte boundary: 395,313,152
max-memory-limit=127666333000
free-chunk-sizes='77, 124, 180, 190, 200, 210, 220, 230, 1000, 2000, 3000, 4000, 5000, 32000, 64000, 257000'
}
}
}".AsLaconicConfig(handling: ConvertErrorHandling.Throw);
using(var app = new ServiceBaseApplication(null, conf))
{
using (var pile = new DefaultPile())
{
pile.Configure(null);
Assert.AreEqual(AllocationMode.FavorSpeed, pile.AllocMode);
Assert.AreEqual(100000, pile.FreeListSize);
Assert.AreEqual(79, pile.MaxSegmentLimit);
Assert.AreEqual(395313152, pile.SegmentSize);
Assert.AreEqual(123666333000, pile.MaxMemoryLimit);
Assert.AreEqual(128, pile.FreeChunkSizes[00]);
Assert.AreEqual(256, pile.FreeChunkSizes[01]);
Assert.AreEqual(512, pile.FreeChunkSizes[02]);
Assert.AreEqual(1024, pile.FreeChunkSizes[03]);
Assert.AreEqual(2000, pile.FreeChunkSizes[04]);
Assert.AreEqual(3000, pile.FreeChunkSizes[05]);
Assert.AreEqual(4000, pile.FreeChunkSizes[06]);
Assert.AreEqual(5000, pile.FreeChunkSizes[07]);
Assert.AreEqual(6000, pile.FreeChunkSizes[08]);
Assert.AreEqual(7000, pile.FreeChunkSizes[09]);
Assert.AreEqual(8000, pile.FreeChunkSizes[10]);
Assert.AreEqual(9000, pile.FreeChunkSizes[11]);
Assert.AreEqual(16000, pile.FreeChunkSizes[12]);
Assert.AreEqual(32000, pile.FreeChunkSizes[13]);
Assert.AreEqual(64000, pile.FreeChunkSizes[14]);
Assert.AreEqual(256000, pile.FreeChunkSizes[15]);
pile.Start();//just to test that it starts ok
}
using (var pile = new DefaultPile("specialNamed"))
{
pile.Configure(null);
Assert.AreEqual(AllocationMode.ReuseSpace, pile.AllocMode);
Assert.AreEqual(99000, pile.FreeListSize);
Assert.AreEqual(73, pile.MaxSegmentLimit);
Assert.AreEqual(395313152, pile.SegmentSize);
Assert.AreEqual(127666333000, pile.MaxMemoryLimit);
Assert.AreEqual(77, pile.FreeChunkSizes[00]);
Assert.AreEqual(124, pile.FreeChunkSizes[01]);
Assert.AreEqual(180, pile.FreeChunkSizes[02]);
Assert.AreEqual(190, pile.FreeChunkSizes[03]);
Assert.AreEqual(200, pile.FreeChunkSizes[04]);
Assert.AreEqual(210, pile.FreeChunkSizes[05]);
Assert.AreEqual(220, pile.FreeChunkSizes[06]);
Assert.AreEqual(230, pile.FreeChunkSizes[07]);
Assert.AreEqual(1000, pile.FreeChunkSizes[08]);
Assert.AreEqual(2000, pile.FreeChunkSizes[09]);
Assert.AreEqual(3000, pile.FreeChunkSizes[10]);
Assert.AreEqual(4000, pile.FreeChunkSizes[11]);
Assert.AreEqual(5000, pile.FreeChunkSizes[12]);
Assert.AreEqual(32000, pile.FreeChunkSizes[13]);
Assert.AreEqual(64000, pile.FreeChunkSizes[14]);
Assert.AreEqual(257000, pile.FreeChunkSizes[15]);
pile.Start();//just to test that it starts ok
}
}//using app
}
public void PutTwo()
{
using (var pile = new DefaultPile())
{
pile.Start();
var ipile = pile as IPile;
var rowIn1 = Charge.MakeFake(new GDID(0, 1));
var rowIn2 = Charge.MakeFake(new GDID(0, 2));
var pp1 = ipile.Put(rowIn1);
var pp2 = ipile.Put(rowIn2);
Assert.AreEqual(2, ipile.ObjectCount);
Assert.AreEqual(DefaultPile.SEG_SIZE_DFLT, ipile.AllocatedMemoryBytes);
Assert.AreEqual(1, ipile.SegmentCount);
var rowOut1 = pile.Get(pp1) as Charge;
var rowOut2 = pile.Get(pp2) as Charge;
Assert.AreEqual(rowIn1, rowOut1);
Assert.AreEqual(rowIn2, rowOut2);
}
}
public static void PutGetDelete_Sequential(int fromSize, int toSize, int fromObjCount, int toObjCount)
{
Console.WriteLine("test will take about 1 minute");
var startTime = DateTime.Now;
var objects = new Dictionary<PilePointer, byte[]>();
var objectCount = 0;
var objectsSumSize = 0;
using (var pile = new DefaultPile() { SegmentSize = PileCacheTestCore.SEG_SIZE })
{
pile.Start();
var dtStop = DateTime.Now.AddMinutes(1);
while (dtStop >= DateTime.Now)
{
// insert routine
var insertCount = NFX.ExternalRandomGenerator.Instance.NextScaledRandomInteger(fromObjCount, toObjCount);
for (int i = 0; i < insertCount; i++)
{
var payloadSize = NFX.ExternalRandomGenerator.Instance.NextScaledRandomInteger(fromSize, toSize);
var payload = new byte[payloadSize];
payload[0] = (byte)NFX.ExternalRandomGenerator.Instance.NextScaledRandomInteger(0, 255);
payload[payloadSize - 1] = (byte)NFX.ExternalRandomGenerator.Instance.NextScaledRandomInteger(0, 255);
var pp = pile.Put(payload);
objects.Add(pp, payload);
objectCount++;
objectsSumSize += payloadSize;
}
// get
if (objectCount > 0)
{
var getCount = ExternalRandomGenerator.Instance.NextScaledRandomInteger(5 * fromObjCount, 5 * toObjCount);
for (int i = 0; i < getCount; i++)
{
var objectIdx = ExternalRandomGenerator.Instance.NextScaledRandomInteger(0, objectCount - 1);
var obj = objects.ElementAt(objectIdx);
var objPayloadFromPile = (byte[])pile.Get(obj.Key);
Assert.AreEqual(obj.Value[0], objPayloadFromPile[0]);
Assert.AreEqual(obj.Value[obj.Value.Length - 1], objPayloadFromPile[obj.Value.Length - 1]);
Assert.AreEqual(obj.Value.Length, objPayloadFromPile.Length);
}
}
// delete
var deleteCount = ExternalRandomGenerator.Instance.NextScaledRandomInteger(fromObjCount, toObjCount);
for (int i = 0; i < deleteCount; i++)
{
if (objectCount == 0) break;
var objectIdx = ExternalRandomGenerator.Instance.NextScaledRandomInteger(0, objectCount - 1);
var obj = objects.ElementAt(objectIdx);
Assert.IsTrue(pile.Delete(obj.Key));
objects.Remove(obj.Key);
objectCount--; objectsSumSize -= obj.Value.Length;
}
}
}
}
public void PileDeleteInMiddleSegment(int payloadSize)
{
using (var pile = new DefaultPile() { SegmentSize = SEG_SIZE})
{
pile.Start();
var pps = new List<PilePointer>();
while (pile.SegmentCount < 4)
pps.Add(pile.Put(generatePayload(payloadSize)));
pile.Delete(pps.Last());
pps.RemoveAt(pps.Count-1);
var objectsInsegmentCount = pps.Count / 3;
Console.WriteLine("{0:N0} object in pile, {1:N0} object per segment", pile.ObjectCount, objectsInsegmentCount);
Assert.AreEqual(3, pile.SegmentCount);
Assert.AreEqual(3, pile.SegmentTotalCount);
for (int i = objectsInsegmentCount; i < 2 * objectsInsegmentCount; i++)
{
pile.Delete(pps[i]);
}
Console.WriteLine("{0:N0} object in pile, {1:N0} segments, {2:N0} segments total", pile.ObjectCount, pile.SegmentCount, pile.SegmentTotalCount);
Assert.AreEqual(2, pile.SegmentCount);
Assert.AreEqual(3, pile.SegmentTotalCount);
}
}
public void PileDeleteInLastSegment(int payloadSize)
{
using (var pile = new DefaultPile() { SegmentSize = SEG_SIZE})
{
pile.Start();
var pps = new List<PilePointer>();
while (pile.SegmentCount < 2)
pps.Add(pile.Put(generatePayload(payloadSize)));
pile.Delete(pps.Last());
pps.RemoveAt(pps.Count-1);
Console.WriteLine("segment count: {0}, segment total count: {1}", pile.SegmentCount, pile.SegmentTotalCount);
Assert.AreEqual(1, pile.SegmentCount);
Assert.AreEqual(1, pile.SegmentTotalCount);
}
}
public void PileSmallObjects(int payloadSize, params int[] freeChunkSizes)
{
using (var pile = new DefaultPile() { SegmentSize = SEG_SIZE, AllocMode = AllocationMode.ReuseSpace })
{
pile.FreeChunkSizes = freeChunkSizes;
pile.Start();
var pps = new List<PilePointer>();
while (pile.SegmentCount < 2)
pps.Add(pile.Put(generatePayload(payloadSize)));
pile.Delete(pps.Last());
pps.RemoveAt(pps.Count-1);
Console.WriteLine("just removed the last added payload and segment should be 1 now, real segment count is {0}", pile.SegmentCount);
var objectsInFirstSegment = pps.Count;
Console.WriteLine("put {0:N0} objects in first segment, pile.ObjectCount {1:N0}", objectsInFirstSegment, pile.ObjectCount);
var deletedObjectCount = 0;
for (int i = 0; i < pps.Count; i+=2, deletedObjectCount++)
Assert.IsTrue(pile.Delete(pps[i]));
Console.WriteLine("deleted {0:N0} objects, pile.ObjectCount {1:N0}", deletedObjectCount, pile.ObjectCount);
var crawlStatus = pile.Crawl(false);
Console.WriteLine("crawl: {0}", crawlStatus);
var pps1 = new List<PilePointer>();
var c = 0;
while (pile.SegmentCount < 3)
{
pps1.Add(pile.Put(generatePayload(payloadSize)));
if (c%20000 ==0 )pile.Crawl(true); //we do crawl because otherwise the 25000 free index xlots get exhausted AND
c++; //this unit tests does not run long enough to cause Crawl within allocator (5+ seconds)
} //so we induce Crawl by hand to rebiild indexes
pile.Delete(pps1.Last());
pps1.RemoveAt(pps1.Count-1);
Console.WriteLine("again just removed the last added payload and segment should be 2 now, real segment count is {0}", pile.SegmentCount);
var objectsInSecondSegment = pps1.Count;
Console.WriteLine("put {0:N0} objects in second segment, pile.ObjectCount {1:N0}", objectsInSecondSegment, pile.ObjectCount);
Assert.AreEqual(objectsInFirstSegment * 2, pile.ObjectCount, "Object count in pile with two full segments must be equal to double object count in full first segment!");
}
}
public static void Put_RandomDelete_ByteArray(int cnt, int durationSec, bool speed, int payloadSizeMin, int payloadSizeMax, int deleteFreq, bool isParallel)
{
using (var pile = new DefaultPile())
{
pile.AllocMode = speed ? AllocationMode.FavorSpeed : AllocationMode.ReuseSpace;
pile.Start();
var startTime = DateTime.UtcNow;
var tasks = new List<Task>();
for (var t = 0; t < (isParallel ? (System.Environment.ProcessorCount - 1) : 1); t++)
tasks.Add(Task.Factory.StartNew(() =>
{
var wlc = 0;
while (true)
{
if ((DateTime.UtcNow - startTime).TotalSeconds >= durationSec) return;
var dict = new Dictionary<int, CheckByteArray>();
Console.WriteLine("Starting a batch of {0}".Args(cnt));
for (int i = 0; i < cnt; i++)
{
var payloadSize = NFX.ExternalRandomGenerator.Instance.NextScaledRandomInteger(payloadSizeMin, payloadSizeMax);
var val = new byte[payloadSize];
val[0] = (byte)NFX.ExternalRandomGenerator.Instance.NextRandomInteger;
val[payloadSize - 1] = (byte)NFX.ExternalRandomGenerator.Instance.NextRandomInteger;
var ptr = pile.Put(val);
var element = new CheckByteArray(ptr, payloadSize - 1, val[0], val[payloadSize - 1]);
dict.Add(i, element);
if (dict.Count > 0 && i % deleteFreq == 0)
{
while (true)
{
var idx = i - NFX.ExternalRandomGenerator.Instance.NextScaledRandomInteger(0, i);
CheckByteArray stored;
if (dict.TryGetValue(idx, out stored))
{
ptr = stored.Ptr;
pile.Delete(ptr);
dict.Remove(idx);
break;
}
}
}
if (dict.Count > 16 && NFX.ExternalRandomGenerator.Instance.NextScaledRandomInteger(0, 100) > 98)
{
var toRead = NFX.ExternalRandomGenerator.Instance.NextScaledRandomInteger(8, 64);
wlc++;
if (wlc % 125 == 0)
Console.WriteLine("Thread {0} is reading {1} elements, total {2}"
.Args(Thread.CurrentThread.ManagedThreadId, toRead, dict.Count));
for (var k = 0; k < toRead; k++)
{
var kvp = dict.Skip(NFX.ExternalRandomGenerator.Instance.NextScaledRandomInteger(0, dict.Count - 1)).First();
var buf = pile.Get(kvp.Value.Ptr) as byte[];
Assert.AreEqual(kvp.Value.FirstByte, buf[0]);
Assert.AreEqual(kvp.Value.LastByte, buf[kvp.Value.IdxLast]);
}
}
}
Console.WriteLine("Thread {0} is doing final read of {1} elements".Args(Thread.CurrentThread.ManagedThreadId, dict.Count));
foreach (var kvp in dict)
{
var buf = pile.Get(kvp.Value.Ptr) as byte[];
Assert.AreEqual(kvp.Value.FirstByte, buf[0]);
Assert.AreEqual(kvp.Value.LastByte, buf[kvp.Value.IdxLast]);
}
}
}, TaskCreationOptions.LongRunning));
Task.WaitAll(tasks.ToArray());
}
}
public static void NoGrowth_TRow(bool speed, int durationSec, int countMin, int countMax)
{
using (var pile = new DefaultPile())
{
pile.AllocMode = speed ? AllocationMode.FavorSpeed : AllocationMode.ReuseSpace;
pile.Start();
var startTime = DateTime.UtcNow;
var tasks = new List<Task>();
for (var t = 0; t < (System.Environment.ProcessorCount - 1); t++)
tasks.Add(Task.Factory.StartNew(() =>
{
var list = new List<CheckTRow>();
bool put = true;
while (true)
{
if ((DateTime.UtcNow - startTime).TotalSeconds >= durationSec) return;
if (put)
{
var cnt = NFX.ExternalRandomGenerator.Instance.NextScaledRandomInteger(countMin, countMax);
for (int j = 0; j < cnt; j++)
{
var val = PersonRow.MakeFake(new GDID());
var ptr = pile.Put(val);
var element = new CheckTRow(ptr, val.ID, val.Address1);
list.Add(element);
}
Console.WriteLine("Thread {0} put {1} objects".Args(Thread.CurrentThread.ManagedThreadId, list.Count));
put = false;
}
else
{
Console.WriteLine("Thread {0} deleted {1} objects".Args(Thread.CurrentThread.ManagedThreadId, list.Count));
for (var j = 0; j < list.Count; j++)
{
var element = list[j];
var buf = pile.Get(element.Ptr) as PersonRow;
Assert.AreEqual(element.Id, buf.ID);
Assert.AreEqual(element.Address, buf.Address1);
pile.Delete(element.Ptr);
}
list.Clear();
put = true;
}
}
}, TaskCreationOptions.LongRunning));
Task.WaitAll(tasks.ToArray());
}
Console.WriteLine("Test finished.");
}
public void PutGetObject()
{
using (var pile = new DefaultPile())
{
pile.Start();
var ipile = pile as IPile;
var rowIn = Charge.MakeFake(new GDID(0, 1));
var pp = ipile.Put(rowIn);
var rowOut = ipile.Get(pp) as Charge;
Assert.AreEqual(rowIn, rowOut);
}
}
public void PutOne()
{
using (var pile = new DefaultPile())
{
pile.Start();
var ipile = pile as IPile;
var row = Charge.MakeFake(new GDID(0, 1));
var pp = ipile.Put(row);
Assert.AreEqual(1, ipile.ObjectCount);
Assert.AreEqual(DefaultPile.SEG_SIZE_DFLT, ipile.AllocatedMemoryBytes);
Assert.AreEqual(1, ipile.SegmentCount);
}
}
public void PutGetRawObject()
{
using (var pile = new DefaultPile())
{
pile.Start();
var ipile = pile as IPile;
var buf = new byte[]{1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0};
var pp = ipile.Put(buf);
byte svr;
var buf2 = ipile.GetRawBuffer(pp, out svr); //main point: we dont get any exceptions
Assert.IsTrue(buf2.Length >= buf.Length);
}
}
public void VarSizes_Checkboard(bool isParallel, int cnt, int minSz, int maxSz, bool speed)
{
using (var pile = new DefaultPile())
{
pile.Start();
pile.AllocMode = speed ? AllocationMode.FavorSpeed : AllocationMode.ReuseSpace;
var tasks = new List<Task>();
for(var t=0; t < (isParallel? (System.Environment.ProcessorCount - 1) : 1); t++)
tasks.Add(
Task.Run( () =>
{
var dict = new Dictionary<PilePointer, dummy>();
var priorOdd = PilePointer.Invalid;
for(var i=0; i<cnt; i++)
{
var even = (i&0x01)==0;
var data = new dummy{ bin = new byte[12 + NFX.ExternalRandomGenerator.Instance.NextScaledRandomInteger(minSz, maxSz)]};
data.bin.WriteBEInt32(0, ExternalRandomGenerator.Instance.NextRandomInteger);
data.bin.WriteBEInt32(data.bin.Length-4, ExternalRandomGenerator.Instance.NextRandomInteger);
var ptr = pile.Put(data);
Assert.IsTrue( ptr.Valid );
if (even)
dict.Add(ptr, data);
else
{
if (priorOdd.Valid)
Assert.IsTrue( pile.Delete(priorOdd) );
priorOdd = ptr;
}
if (i%1000==0)
Console.WriteLine("Thread{0} did {1}; allocated {2} bytes, utilized {3} bytes by {4} objects {5} bytes/obj. ",
Thread.CurrentThread.ManagedThreadId,
i,
pile.AllocatedMemoryBytes,
pile.UtilizedBytes,
pile.ObjectCount,
pile.UtilizedBytes / pile.ObjectCount);
}
Console.WriteLine("Thread {0} Population done, now checking the buffers... {1}",Thread.CurrentThread.ManagedThreadId, DateTime.Now);
foreach(var entry in dict)
Assert.IsTrue( NFX.IOMiscUtils.MemBufferEquals(entry.Value.bin, (pile.Get(entry.Key) as dummy).bin));
Console.WriteLine("Thread {0} DONE. {1}", Thread.CurrentThread.ManagedThreadId, DateTime.Now);
})
);//add
Task.WaitAll( tasks.ToArray() );
}
}
public void DeleteExisting()
{
using (var pile = new DefaultPile())
{
pile.Start();
var ipile = pile as IPile;
var rowIn = Charge.MakeFake(new GDID(0, 1));
var pp = ipile.Put(rowIn);
ipile.Delete(pp);
Assert.AreEqual(0, ipile.ObjectCount);
Assert.AreEqual(0, ipile.AllocatedMemoryBytes);
Assert.AreEqual(0, ipile.UtilizedBytes);
Assert.AreEqual(0, ipile.OverheadBytes);
Assert.AreEqual(0, ipile.SegmentCount);
var rowOut = ipile.Get(pp);
}
}
public void VarSizes_Increasing_Random(bool isParallel, int cnt, int minSz, int maxSz, bool speed, bool rnd)
{
using (var pile = new DefaultPile())
{
pile.Start();
pile.AllocMode = speed ? AllocationMode.FavorSpeed : AllocationMode.ReuseSpace;
var sw = Stopwatch.StartNew();
var tasks = new List<Task>();
for(var t=0; t < (isParallel? (System.Environment.ProcessorCount - 1) : 1); t++)
tasks.Add(
Task.Run( () =>
{
var dict = new Dictionary<PilePointer, dummy>();
var lst = new List<PilePointer>();
var priorOdd = PilePointer.Invalid;
for(var i=0; i<cnt; i++)
{
var buf = new byte[12 +
minSz+
(
rnd
? (NFX.ExternalRandomGenerator.Instance.NextScaledRandomInteger(0, (int)(maxSz*(i/(double)cnt))))
: (int)(maxSz*(i/(double)cnt))
)
];
var data = new dummy{ bin = buf};
data.bin.WriteBEInt32(0, ExternalRandomGenerator.Instance.NextRandomInteger);
data.bin.WriteBEInt32(data.bin.Length-4, ExternalRandomGenerator.Instance.NextRandomInteger);
var ptr = pile.Put(data);
Assert.IsTrue( ptr.Valid );
dict.Add(ptr, data);
lst.Add(ptr);
if (i>cnt/3)
{
if (ExternalRandomGenerator.Instance.NextRandomInteger > 0)
{
var ri = ExternalRandomGenerator.Instance.NextScaledRandomInteger(0, lst.Count-1);
var pp = lst[ri];
if (!pp.Valid) continue;
Assert.IsTrue( pile.Delete(pp) );
dict.Remove( pp );
lst[ri] = PilePointer.Invalid;
}
}
if (i%1000==0)
Console.WriteLine("Thread{0} did {1}; allocated {2} bytes, utilized {3} bytes by {4} objects {5} bytes/obj. ",
Thread.CurrentThread.ManagedThreadId,
i,
pile.AllocatedMemoryBytes,
pile.UtilizedBytes,
pile.ObjectCount,
pile.UtilizedBytes / pile.ObjectCount);
}
Console.WriteLine("Thread {0} Population done, now checking the buffers... {1}",Thread.CurrentThread.ManagedThreadId, DateTime.Now);
foreach(var entry in dict)
Assert.IsTrue( NFX.IOMiscUtils.MemBufferEquals(entry.Value.bin, (pile.Get(entry.Key) as dummy).bin));
Console.WriteLine("Thread {0} DONE. {1}", Thread.CurrentThread.ManagedThreadId, DateTime.Now);
})
);//add
Task.WaitAll( tasks.ToArray() );
var el = sw.ElapsedMilliseconds;
var gt = cnt * tasks.Count;
Console.WriteLine("Total objects: {0:n0} in {1:n0} ms at {2:n0} obj/sec".Args(gt, el, gt / (el / 1000d)));
}
}
public void DeleteInvalid()
{
using (var pile = new DefaultPile())
{
pile.Start();
var ipile = pile as IPile;
ipile.Delete(PilePointer.Invalid);
}
}
public void Parallel_PutGetDelete_Random()
{
const int PUTTER_CNT = 2, PUTTER_OP_CNT = 2 * 10000;
const int GETTER_CNT = 6, GETTER_OP_CNT = 2 * 30000;
const int DELETER_CNT = 2, DELETER_OP_CNT = 2 * 10000;
var data = new ConcurrentDictionary<PilePointer, string>();
var getAccessViolations = new ConcurrentDictionary<int, int>();
var deleteAccessViolations = new ConcurrentDictionary<int, int>();
using (var pile = new DefaultPile())
{
pile.Start();
var ipile = pile as IPile;
// putter tasks
var putters = new Task[PUTTER_CNT];
for (int it = 0; it < PUTTER_CNT; it++)
{
var task = new Task(() => {
for (int i = 0; i < PUTTER_OP_CNT; i++)
{
var str = NFX.Parsing.NaturalTextGenerator.Generate();
var pp = ipile.Put(str);
data.TryAdd(pp, str);
}
});
putters[it] = task;
}
// getter tasks
var getters = new Task[GETTER_CNT];
for (int it = 0; it < GETTER_CNT; it++)
{
var task = new Task(() => {
for (int i = 0; i < GETTER_OP_CNT; i++)
{
if (data.Count == 0) {
System.Threading.Thread.Yield();
continue;
}
var idx = ExternalRandomGenerator.Instance.NextScaledRandomInteger(0, data.Count-1);
var kvp = data.ElementAt(idx);
try
{
var str = ipile.Get(kvp.Key);
Assert.AreEqual(str, kvp.Value);
}
catch (PileAccessViolationException)
{
getAccessViolations.AddOrUpdate(System.Threading.Thread.CurrentThread.ManagedThreadId, 1, (mid, val) => val + 1);
}
}
});
getters[it] = task;
}
// deleter tasks
var deleters = new Task[DELETER_CNT];
for (int it = 0; it < DELETER_CNT; it++)
{
var task = new Task(() => {
for (int i = 0; i < DELETER_OP_CNT; i++)
{
if (data.Count == 0) {
System.Threading.Thread.Yield();
continue;
}
var idx = ExternalRandomGenerator.Instance.NextScaledRandomInteger(0, data.Count-1);
var kvp = data.ElementAt(idx);
try
{
ipile.Delete(kvp.Key);
}
catch (PileAccessViolationException)
{
deleteAccessViolations.AddOrUpdate(System.Threading.Thread.CurrentThread.ManagedThreadId, 1, (mid, val) => val + 1);
}
}
});
deleters[it] = task;
}
foreach (var task in putters) task.Start();
foreach (var task in getters) task.Start();
foreach (var task in deleters) task.Start();
Task.WaitAll(putters.Concat(getters).Concat(deleters).ToArray());
foreach (var kvp in getAccessViolations)
Console.WriteLine("Get thread '{0}' {1:n0} times accessed deleted pointer", kvp.Key, kvp.Value);
foreach (var kvp in deleteAccessViolations)
Console.WriteLine("Del thread '{0}' {1:n0} times accessed deleted pointer", kvp.Key, kvp.Value);
}
}
public void GetNoObject()
{
using (var pile = new DefaultPile())
{
pile.Start();
var ipile = pile as IPile;
ipile.Get(PilePointer.Invalid);
}
}
public void Purge()
{
using (var pile = new DefaultPile())
{
pile.Start();
var ipile = pile as IPile;
var rowIn = Charge.MakeFake(new GDID(0, 1));
var pp = ipile.Put(rowIn);
ipile.Purge();
Assert.AreEqual(0, ipile.ObjectCount);
Assert.AreEqual(0, ipile.SegmentCount);
var rowOut = ipile.Get(pp);
}
}
public static void Chessboard_ByteArray(bool speed, int durationSec, int payloadSizeMin, int payloadSizeMax, bool isParallel)
{
using (var pile = new DefaultPile())
{
pile.AllocMode = speed ? AllocationMode.FavorSpeed : AllocationMode.ReuseSpace;
pile.Start();
var startTime = DateTime.UtcNow;
var tasks = new List<Task>();
for (var t = 0; t < (isParallel ? (System.Environment.ProcessorCount - 1) : 1); t++)
tasks.Add(Task.Factory.StartNew(() =>
{
var list = new List<CheckByteArray>();
var i = 0;
var wlc = 0;
while (true)
{
if ((DateTime.UtcNow - startTime).TotalSeconds >= durationSec) break;
var payloadSize = NFX.ExternalRandomGenerator
.Instance.NextScaledRandomInteger(payloadSizeMin, payloadSizeMax);
var val = new byte[payloadSize];
val[0] = (byte)NFX.ExternalRandomGenerator.Instance.NextRandomInteger;
val[payloadSize - 1] = (byte)NFX.ExternalRandomGenerator.Instance.NextRandomInteger;
var ptr = pile.Put(val);
var element = new CheckByteArray(ptr, payloadSize - 1, val[0], val[payloadSize - 1]);
list.Add(element);
// delete previous element
if (list.Count > 1 && i % 2 == 0)
{
ptr = list[list.Count - 2].Ptr;
pile.Delete(ptr);
list.RemoveAt(list.Count - 2);
}
// get several random elements
if (list.Count > 64 && NFX.ExternalRandomGenerator.Instance.NextScaledRandomInteger(0, 100) > 98)
{
var toRead = NFX.ExternalRandomGenerator.Instance.NextScaledRandomInteger(8, 64);
wlc++;
if (wlc % 125 == 0)
Console.WriteLine("Thread {0} is reading {1} elements, total {2}, Pile objects {3}, Pile segments {4} Pile Bytes {5}"
.Args(Thread.CurrentThread.ManagedThreadId, toRead, list.Count, pile.ObjectCount, pile.SegmentCount, pile.AllocatedMemoryBytes));
for (var k = 0; k < toRead; k++)
{
element = list[NFX.ExternalRandomGenerator.Instance.NextScaledRandomInteger(0, list.Count - 1)];
var buf = pile.Get(element.Ptr) as byte[];
Assert.AreEqual(element.FirstByte, buf[0]);
Assert.AreEqual(element.LastByte, buf[element.IdxLast]);
}
}
if (i == Int32.MaxValue)
i = 0;
else
i++;
if (list.Count == Int32.MaxValue)
list = new List<CheckByteArray>();
}
// total check
Console.WriteLine("Thread {0} is doing final read of {1} elements, ObjectCount {2}"
.Args(Thread.CurrentThread.ManagedThreadId, list.Count, pile.ObjectCount));
foreach (var element in list)
{
var buf = pile.Get(element.Ptr) as byte[];
Assert.AreEqual(element.FirstByte, buf[0]);
Assert.AreEqual(element.LastByte, buf[element.IdxLast]);
}
return;
}, TaskCreationOptions.LongRunning));
Task.WaitAll(tasks.ToArray());
}
}
