public void FitPreallocateString()
{
using (var pile = new DefaultPile())
{
pile.Start();
var obj1 = "abcdefgh";
var p1 = pile.Put(obj1, preallocateBlockSize: 4000);
var got = pile.Get(p1) as string;
Aver.AreEqual("abcdefgh", got);
Aver.AreEqual(4000, pile.SizeOf(p1));
var obj2 = "abcdefghijklmnopqrst0912345";
Aver.IsTrue(pile.Put(p1, obj2));
Aver.AreEqual(1, pile.ObjectCount);
got = pile.Get(p1) as string;
Aver.AreEqual("abcdefghijklmnopqrst0912345", got);
Aver.IsTrue(pile.AllocatedMemoryBytes > 0);
Aver.IsTrue(pile.Delete(p1));
Aver.IsTrue(pile.AllocatedMemoryBytes == 0);
Aver.IsTrue(pile.ObjectCount == 0);
}
}
public void LinkStringNoPreallocate()
{
using (var pile = new DefaultPile(NOPApplication.Instance))
{
pile.Start();
var obj1 = "abcdefgh";
var p1 = pile.Put(obj1);
var got = pile.Get(p1) as string;
Aver.AreEqual("abcdefgh", got);
var obj2 = "abcdefghijklmnopqrst0912345";
Aver.IsTrue(pile.Put(p1, obj2));
Aver.AreEqual(1, pile.ObjectCount);
got = pile.Get(p1) as string;
Aver.AreEqual("abcdefghijklmnopqrst0912345", got);
Aver.IsTrue(pile.AllocatedMemoryBytes > 0);
Aver.IsTrue(pile.Delete(p1));
Aver.IsTrue(pile.AllocatedMemoryBytes == 0);
Aver.IsTrue(pile.ObjectCount == 0);
}
}
public void FitPreallocateByteArray()
{
using (var pile = new DefaultPile())
{
pile.Start();
var obj1 = new byte[3];
var p1 = pile.Put(obj1, preallocateBlockSize: 4000);
var got = pile.Get(p1) as byte[];
Aver.AreEqual(3, got.Length);
Aver.AreEqual(4000, pile.SizeOf(p1));
var obj2 = new byte[571];
Aver.IsTrue(pile.Put(p1, obj2));
Aver.AreEqual(1, pile.ObjectCount);
got = pile.Get(p1) as byte[];
Aver.AreEqual(571, got.Length);
Aver.IsTrue(pile.AllocatedMemoryBytes > 0);
Aver.IsTrue(pile.Delete(p1));
Aver.IsTrue(pile.AllocatedMemoryBytes == 0);
Aver.IsTrue(pile.ObjectCount == 0);
}
}
public void PutGetTwo()
{
using (var pile = new DefaultPile())
{
pile.Start();
var ipile = pile as IPile;
var rowIn1 = CheckoutRow.MakeFake(new GDID(0, 1));
var rowIn2 = CheckoutRow.MakeFake(new GDID(0, 2));
var pp1 = ipile.Put(rowIn1);
var pp2 = ipile.Put(rowIn2);
Assert.AreEqual(2, ipile.ObjectCount);
Assert.AreEqual(DefaultPile.SEG_SIZE_DFLT, ipile.AllocatedMemoryBytes);
Assert.AreEqual(1, ipile.SegmentCount);
var rowOut1 = pile.Get(pp1) as CheckoutRow;
var rowOut2 = pile.Get(pp2) as CheckoutRow;
Assert.AreEqual(rowIn1, rowOut1);
Assert.AreEqual(rowIn2, rowOut2);
}
}
public void PutSubarray()
{
using (var pile = new DefaultPile(NOPApplication.Instance))
{
pile.Start();
var src = new byte[] { 1, 6, 9, 250 };
var sub = new Subarray <byte>();
sub.Set(src, 2);
var ptr1 = pile.Put(sub);
sub.Set(src, 4);
var ptr2 = pile.Put(sub);
var got1 = pile.Get(ptr1) as byte[];
var got2 = pile.Get(ptr2) as byte[];
Aver.IsNotNull(got1);
Aver.AreEqual(2, got1.Length);
Aver.AreEqual(1, got1[0]);
Aver.AreEqual(6, got1[1]);
Aver.IsNotNull(got2);
Aver.AreEqual(4, got2.Length);
Aver.AreEqual(1, got2[0]);
Aver.AreEqual(6, got2[1]);
Aver.AreEqual(9, got2[2]);
Aver.AreEqual(250, got2[3]);
}
}
public void LinkByteArrayNoPreallocate()
{
using (var pile = new DefaultPile())
{
pile.Start();
var obj1 = new byte[12];
var p1 = pile.Put(obj1);
var got = pile.Get(p1) as byte[];
Aver.AreEqual(12, got.Length);
var obj2 = new byte[389];
Aver.IsTrue(pile.Put(p1, obj2));
Aver.AreEqual(1, pile.ObjectCount);
got = pile.Get(p1) as byte[];
Aver.AreEqual(389, got.Length);
Aver.IsTrue(pile.AllocatedMemoryBytes > 0);
Aver.IsTrue(pile.Delete(p1));
Aver.IsTrue(pile.AllocatedMemoryBytes == 0);
Aver.IsTrue(pile.ObjectCount == 0);
}
}
public void Buffers(int count, int segmentSize)
{
using (var pile = new DefaultPile())
{
pile.SegmentSize = segmentSize;
pile.Start();
var hs = new HashSet <int>();
for (var i = 0; i < count; i++)
{
var buf = new byte[4 + (i % 512)];
buf.WriteBEInt32(i);
pile.Put(buf);
hs.Add(i);
}
Console.WriteLine("Created {0} segments".Args(pile.SegmentCount));
var j = 0;
foreach (var entry in pile)
{
var buf = pile.Get(entry.Pointer) as byte[];
Aver.IsNotNull(buf);
Aver.IsTrue(buf.Length >= 4);
var i = buf.ReadBEInt32();
Aver.IsTrue(hs.Remove(i));
Aver.IsTrue(entry.Type == PileEntry.DataType.Buffer);
j++;
}
Aver.AreEqual(j, count);
Aver.AreEqual(0, hs.Count);
}
}
[Run("cnt=32000000 size=32 parallel=12")] //12 threads put 32,000,000 byte[32] in 5,008 msec at 6,389,776 ops/sec
//12 threads got 32,000,000 byte[32] in 3,787 msec at 8,449,960 ops/sec
//--------------------------------------------------------------------------------
//12 threads put 32,000,000 byte[32] in 2,963 msec at 10,799,865 ops/sec
//12 threads got 32,000,000 byte[32] in 2,253 msec at 14,203,285 ops/sec
public void ByteBuf(int CNT, int size, int parallel)
{
using (var pile = new DefaultPile(NOPApplication.Instance))
{
pile.Start();
var str = new byte[size];
var pp = pile.Put(str);
var sw = Stopwatch.StartNew();
Parallel.For(0, CNT, new ParallelOptions {
MaxDegreeOfParallelism = parallel
},
(i) =>
{
pile.Put(str);
});
var el = sw.ElapsedMilliseconds;
Console.WriteLine("{0:n0} threads put {1:n0} byte[{2:n0}] in {3:n0} msec at {4:n0} ops/sec".Args(parallel, CNT, size, el, CNT / (el / 1000d)));
sw.Restart();
Parallel.For(0, CNT, new ParallelOptions {
MaxDegreeOfParallelism = parallel
},
(i) =>
{
pile.Get(pp);
});
el = sw.ElapsedMilliseconds;
Console.WriteLine("{0:n0} threads got {1:n0} byte[{2:n0}] in {3:n0} msec at {4:n0} ops/sec".Args(parallel, CNT, size, el, CNT / (el / 1000d)));
}
}
public void LinkNoPreallocate()
{
using (var pile = new DefaultPile())
{
pile.Start();
var obj1 = new Payload {
ID = 1, Name = "1", Data = null
};
var p1 = pile.Put(obj1);
Aver.IsTrue(pile.SizeOf(p1) < 128);
var obj2 = new Payload {
ID = 2, Name = "2", Data = new byte [128]
};
Aver.IsTrue(pile.Put(p1, obj2));
Aver.AreEqual(1, pile.ObjectCount);
var got = pile.Get(p1) as Payload;
Aver.AreEqual(2, got.ID);
Aver.AreEqual("2", got.Name);
Aver.IsNotNull(got.Data);
Aver.AreEqual(128, got.Data.Length);
Aver.IsTrue(pile.AllocatedMemoryBytes > 0);
Aver.IsTrue(pile.Delete(p1));
Aver.IsTrue(pile.AllocatedMemoryBytes == 0);
Aver.IsTrue(pile.ObjectCount == 0);
}
}
public void FitPreallocate()
{
using (var pile = new DefaultPile())
{
pile.Start();
var obj1 = new Payload {
ID = 1, Name = "1", Data = null
};
var p1 = pile.Put(obj1, preallocateBlockSize: 4000);
Aver.AreEqual(4000, pile.SizeOf(p1));
var obj2 = new Payload {
ID = 2, Name = "2", Data = new byte [] { 1, 2, 3, 4, 5, 6, 7, 8 }
};
Aver.IsTrue(pile.Put(p1, obj2));
Aver.AreEqual(1, pile.ObjectCount);
var got = pile.Get(p1) as Payload;
Aver.AreEqual(2, got.ID);
Aver.AreEqual("2", got.Name);
Aver.IsNotNull(got.Data);
Aver.AreEqual(8, got.Data.Length);
Aver.IsTrue(pile.AllocatedMemoryBytes > 0);
Aver.IsTrue(pile.Delete(p1));
Aver.IsTrue(pile.AllocatedMemoryBytes == 0);
Aver.IsTrue(pile.ObjectCount == 0);
}
}
public void LinkPreallocate()
{
using (var pile = new DefaultPile(NOPApplication.Instance))
{
pile.Start();
var obj1 = new Payload {
ID = 1, Name = "1", Data = null
};
var p1 = pile.Put(obj1, preallocateBlockSize: 2000);
Aver.AreEqual(2000, pile.SizeOf(p1));
var obj2 = new Payload {
ID = 2, Name = "2", Data = new byte [3000]
};
Aver.IsTrue(pile.Put(p1, obj2));
Aver.AreEqual(1, pile.ObjectCount);
var got = pile.Get(p1) as Payload;
Aver.AreEqual(2, got.ID);
Aver.AreEqual("2", got.Name);
Aver.IsNotNull(got.Data);
Aver.AreEqual(3000, got.Data.Length);
Aver.IsTrue(pile.AllocatedMemoryBytes > 0);
Aver.IsTrue(pile.Delete(p1));
Aver.IsTrue(pile.AllocatedMemoryBytes == 0);
Aver.IsTrue(pile.ObjectCount == 0);
}
}
public static void NoGrowth_ByteArray(bool speed, int durationSec, int payloadSizeMin, int payloadSizeMax, int countMin, int countMax)
{
using (var pile = new DefaultPile(NOPApplication.Instance))
{
pile.AllocMode = speed ? AllocationMode.FavorSpeed : AllocationMode.ReuseSpace;
pile.Start();
var startTime = DateTime.UtcNow;
var tasks = new List <Task>();
for (var t = 0; t < (System.Environment.ProcessorCount - 1); t++)
{
tasks.Add(Task.Factory.StartNew(() =>
{
var list = new List <CheckByteArray>();
bool put = true;
while (true)
{
if ((DateTime.UtcNow - startTime).TotalSeconds >= durationSec)
{
return;
}
if (put)
{
var cnt = Ambient.Random.NextScaledRandomInteger(countMin, countMax);
for (int j = 0; j < cnt; j++)
{
var payloadSize = Ambient.Random.NextScaledRandomInteger(payloadSizeMin, payloadSizeMax);
var val = new byte[payloadSize];
val[0] = (byte)Ambient.Random.NextRandomInteger;
val[payloadSize - 1] = (byte)Ambient.Random.NextRandomInteger;
var ptr = pile.Put(val);
var element = new CheckByteArray(ptr, payloadSize - 1, val[0], val[payloadSize - 1]);
list.Add(element);
}
Console.WriteLine("Thread {0} put {1} objects".Args(Thread.CurrentThread.ManagedThreadId, list.Count));
put = false;
}
else
{
Console.WriteLine("Thread {0} deleted {1} objects".Args(Thread.CurrentThread.ManagedThreadId, list.Count));
for (var j = 0; j < list.Count; j++)
{
var element = list[j];
var buf = pile.Get(element.Ptr) as byte[];
Aver.AreEqual(element.FirstByte, buf[0]);
Aver.AreEqual(element.LastByte, buf[element.IdxLast]);
pile.Delete(element.Ptr);
}
list.Clear();
put = true;
}
}
}, TaskCreationOptions.LongRunning));
}
Task.WaitAll(tasks.ToArray());
}
Console.WriteLine("Test finished.");
}
public static void NoGrowth_TRow(bool speed, int durationSec, int countMin, int countMax)
{
using (var pile = new DefaultPile())
{
pile.AllocMode = speed ? AllocationMode.FavorSpeed : AllocationMode.ReuseSpace;
pile.Start();
var startTime = DateTime.UtcNow;
var tasks = new List <Task>();
for (var t = 0; t < (System.Environment.ProcessorCount - 1); t++)
{
tasks.Add(Task.Factory.StartNew(() =>
{
var list = new List <CheckTRow>();
bool put = true;
while (true)
{
if ((DateTime.UtcNow - startTime).TotalSeconds >= durationSec)
{
return;
}
if (put)
{
var cnt = NFX.ExternalRandomGenerator.Instance.NextScaledRandomInteger(countMin, countMax);
for (int j = 0; j < cnt; j++)
{
var val = PersonRow.MakeFake(new GDID());
var ptr = pile.Put(val);
var element = new CheckTRow(ptr, val.ID, val.Address1);
list.Add(element);
}
Console.WriteLine("Thread {0} put {1} objects".Args(Thread.CurrentThread.ManagedThreadId, list.Count));
put = false;
}
else
{
Console.WriteLine("Thread {0} deleted {1} objects".Args(Thread.CurrentThread.ManagedThreadId, list.Count));
for (var j = 0; j < list.Count; j++)
{
var element = list[j];
var buf = pile.Get(element.Ptr) as PersonRow;
Aver.AreEqual(element.Id, buf.ID);
Aver.AreEqual(element.Address, buf.Address1);
pile.Delete(element.Ptr);
}
list.Clear();
put = true;
}
}
}, TaskCreationOptions.LongRunning));
}
Task.WaitAll(tasks.ToArray());
}
Console.WriteLine("Test finished.");
}
public void MulticulturalString()
{
using (var pile = new DefaultPile(NOPApplication.Instance))
{
pile.Start();
var str = @"
外国語の学習と教授
Language Learning and Teaching
Изучение и обучение иностранных языков
Tere Daaheng Aneng Karimah
語文教學・语文教学
Enseñanza y estudio de idiomas
Изучаване и Преподаване на Чужди Езици
ქართული ენის შესწავლა და სწავლება
'læŋɡwidʒ 'lɘr:niŋ ænd 'ti:tʃiŋ
Lus kawm thaib qhia
Ngôn Ngữ, Sự học,
ללמוד וללמד את השֵפה
L'enseignement et l'étude des langues
말배우기와 가르치기
Nauka języków obcych
Γλωσσική Εκμὰθηση και Διδασκαλία
ﺗﺪﺭﯾﺲ ﻭ ﯾﺎﺩﮔﯿﺮﯼ ﺯﺑﺎﻥ
Sprachlernen und -lehren
ﺗﻌﻠﻢ ﻭﺗﺪﺭﻳﺲ ﺍﻟﻌﺮﺑﻴﺔ
เรียนและสอนภาษา";
var pp = pile.Put(str);
var got = pile.Get(pp) as string;
Aver.AreEqual(str, got);
}
}
public void PutString()
{
using (var pile = new DefaultPile(NOPApplication.Instance))
{
pile.Start();
var ptr = pile.Put("abcd123");
var got = pile.Get(ptr) as string;
Aver.IsNotNull(got);
Aver.AreEqual("abcd123", got);
}
}
private void btnPersonGet_Click(object sender, EventArgs e)
{
if (lbPerson.SelectedItem == null)
{
return;
}
var pp = (PilePointer)lbPerson.SelectedItem;
var cnt = tbPersonCount.Text.AsInt(10);
var w = Stopwatch.StartNew();
object person = null;
for (var i = 0; i < cnt; i++)
{
person = m_Pile.Get(pp);
}
var elps = w.ElapsedMilliseconds;
MessageBox.Show("Read {0:n0} in {1:n0}ms at {2:n0}/sec \r\n {3}".Args(cnt, elps, cnt / (elps / 1000d), person == null ? "[null]" : person.ToString()));
}
public void PutByteArray()
{
using (var pile = new DefaultPile(NOPApplication.Instance))
{
pile.Start();
var ptr = pile.Put(new byte[] { 1, 6, 9 });
var got = pile.Get(ptr) as byte[];
Aver.IsNotNull(got);
Aver.AreEqual(3, got.Length);
Aver.AreEqual(1, got[0]);
Aver.AreEqual(6, got[1]);
Aver.AreEqual(9, got[2]);
}
}
[Run("cnt=1048576")] // 1024 * 1024
public void LongMulticulturalString(int cnt)
{
using (var pile = new DefaultPile(NOPApplication.Instance))
{
pile.Start();
var str = new string('久', cnt);
var pp = pile.Put(str);
var got = pile.Get(pp) as string;
Aver.AreEqual(str, got);
}
}
public void StringCorrectess(int len, int deleteEvery, int parallel)
{
using (var pile = new DefaultPile())
{
pile.Start();
var bag = new ConcurrentBag <PilePointer>();
var deleted = 0;
Parallel.For(0, len, new ParallelOptions {
MaxDegreeOfParallelism = parallel
},
(i) =>
{
var str = new string('a', i);
PilePointer pp = PilePointer.Invalid;
if (i % 2 == 0)
{
if (bag.TryTake(out pp))
{
pile.Put(pp, str);
}
}
pp = pile.Put(str, preallocateBlockSize: i + 100);
var got = pile.Get(pp) as string;
Aver.AreEqual(str, got);
if (i % deleteEvery == 0)
{
PilePointer dp;
if (bag.TryTake(out dp))
{
if (pile.Delete(dp, false))
{
Interlocked.Increment(ref deleted);
}
}
bag.Add(pp);
}
});
Console.WriteLine("Deleted {0:n0}", deleted);
}
}
public void EmptyTest()
{
using (var m_pile = new DefaultPile()
{
AllocMode = AllocationMode.FavorSpeed
})
{
m_pile.Start();
var test = m_pile.Put("TEST");
Console.WriteLine(test);
Console.WriteLine(m_pile.Get(test));
m_pile.WaitForCompleteStop();
}
Aver.IsTrue(true);
}
public void ByteCorrectess(int len, int deleteEvery, int parallel)
{
using (var pile = new DefaultPile(NOPApplication.Instance))
{
pile.Start();
var bag = new ConcurrentBag <PilePointer>();
var deleted = 0;
Parallel.For(0, len, new ParallelOptions {
MaxDegreeOfParallelism = parallel
},
(i) =>
{
var original = new byte[len - i];
var pp = pile.Put(original);
var got = pile.Get(pp) as byte[];
Aver.AreEqual(original.Length, got.Length);
if (i % deleteEvery == 0)
{
PilePointer dp;
if (bag.TryTake(out dp))
{
if (pile.Delete(dp, false))
{
Interlocked.Increment(ref deleted);
}
}
bag.Add(pp);
}
});
Console.WriteLine("Deleted {0:n0}", deleted);
}
}
public static void Chessboard_ByteArray(bool speed, int durationSec, int payloadSizeMin, int payloadSizeMax, bool isParallel)
{
using (var pile = new DefaultPile())
{
pile.AllocMode = speed ? AllocationMode.FavorSpeed : AllocationMode.ReuseSpace;
pile.Start();
var startTime = DateTime.UtcNow;
var tasks = new List<Task>();
for (var t = 0; t < (isParallel ? (System.Environment.ProcessorCount - 1) : 1); t++)
tasks.Add(Task.Factory.StartNew(() =>
{
var list = new List<CheckByteArray>();
var i = 0;
var wlc = 0;
while (true)
{
if ((DateTime.UtcNow - startTime).TotalSeconds >= durationSec) break;
var payloadSize = NFX.ExternalRandomGenerator
.Instance.NextScaledRandomInteger(payloadSizeMin, payloadSizeMax);
var val = new byte[payloadSize];
val[0] = (byte)NFX.ExternalRandomGenerator.Instance.NextRandomInteger;
val[payloadSize - 1] = (byte)NFX.ExternalRandomGenerator.Instance.NextRandomInteger;
var ptr = pile.Put(val);
var element = new CheckByteArray(ptr, payloadSize - 1, val[0], val[payloadSize - 1]);
list.Add(element);
// delete previous element
if (list.Count > 1 && i % 2 == 0)
{
ptr = list[list.Count - 2].Ptr;
pile.Delete(ptr);
list.RemoveAt(list.Count - 2);
}
// get several random elements
if (list.Count > 64 && NFX.ExternalRandomGenerator.Instance.NextScaledRandomInteger(0, 100) > 98)
{
var toRead = NFX.ExternalRandomGenerator.Instance.NextScaledRandomInteger(8, 64);
wlc++;
if (wlc % 125 == 0)
Console.WriteLine("Thread {0} is reading {1} elements, total {2}, Pile objects {3}, Pile segments {4} Pile Bytes {5}"
.Args(Thread.CurrentThread.ManagedThreadId, toRead, list.Count, pile.ObjectCount, pile.SegmentCount, pile.AllocatedMemoryBytes));
for (var k = 0; k < toRead; k++)
{
element = list[NFX.ExternalRandomGenerator.Instance.NextScaledRandomInteger(0, list.Count - 1)];
var buf = pile.Get(element.Ptr) as byte[];
Assert.AreEqual(element.FirstByte, buf[0]);
Assert.AreEqual(element.LastByte, buf[element.IdxLast]);
}
}
if (i == Int32.MaxValue)
i = 0;
else
i++;
if (list.Count == Int32.MaxValue)
list = new List<CheckByteArray>();
}
// total check
Console.WriteLine("Thread {0} is doing final read of {1} elements, ObjectCount {2}"
.Args(Thread.CurrentThread.ManagedThreadId, list.Count, pile.ObjectCount));
foreach (var element in list)
{
var buf = pile.Get(element.Ptr) as byte[];
Assert.AreEqual(element.FirstByte, buf[0]);
Assert.AreEqual(element.LastByte, buf[element.IdxLast]);
}
return;
}, TaskCreationOptions.LongRunning));
Task.WaitAll(tasks.ToArray());
}
}
public static void PutGetDelete_Sequential(int fromSize, int toSize, int fromObjCount, int toObjCount)
{
Console.WriteLine("test will take about 1 minute");
var startTime = DateTime.Now;
var objects = new Dictionary <PilePointer, byte[]>();
var objectCount = 0;
var objectsSumSize = 0;
using (var pile = new DefaultPile()
{
SegmentSize = PileCacheTestCore.SEG_SIZE
})
{
pile.Start();
var dtStop = DateTime.Now.AddMinutes(1);
while (dtStop >= DateTime.Now)
{
// insert routine
var insertCount = NFX.ExternalRandomGenerator.Instance.NextScaledRandomInteger(fromObjCount, toObjCount);
for (int i = 0; i < insertCount; i++)
{
var payloadSize = NFX.ExternalRandomGenerator.Instance.NextScaledRandomInteger(fromSize, toSize);
var payload = new byte[payloadSize];
payload[0] = (byte)NFX.ExternalRandomGenerator.Instance.NextScaledRandomInteger(0, 255);
payload[payloadSize - 1] = (byte)NFX.ExternalRandomGenerator.Instance.NextScaledRandomInteger(0, 255);
var pp = pile.Put(payload);
objects.Add(pp, payload);
objectCount++;
objectsSumSize += payloadSize;
}
// get
if (objectCount > 0)
{
var getCount = ExternalRandomGenerator.Instance.NextScaledRandomInteger(5 * fromObjCount, 5 * toObjCount);
for (int i = 0; i < getCount; i++)
{
var objectIdx = ExternalRandomGenerator.Instance.NextScaledRandomInteger(0, objectCount - 1);
var obj = objects.ElementAt(objectIdx);
var objPayloadFromPile = (byte[])pile.Get(obj.Key);
Assert.AreEqual(obj.Value[0], objPayloadFromPile[0]);
Assert.AreEqual(obj.Value[obj.Value.Length - 1], objPayloadFromPile[obj.Value.Length - 1]);
Assert.AreEqual(obj.Value.Length, objPayloadFromPile.Length);
}
}
// delete
var deleteCount = ExternalRandomGenerator.Instance.NextScaledRandomInteger(fromObjCount, toObjCount);
for (int i = 0; i < deleteCount; i++)
{
if (objectCount == 0)
{
break;
}
var objectIdx = ExternalRandomGenerator.Instance.NextScaledRandomInteger(0, objectCount - 1);
var obj = objects.ElementAt(objectIdx);
Assert.IsTrue(pile.Delete(obj.Key));
objects.Remove(obj.Key);
objectCount--; objectsSumSize -= obj.Value.Length;
}
}
}
}
public void TestThreadSafety(int cnt, int tcount, int seconds)
{
using (var pile = new DefaultPile())
{
// pile.SegmentSize = 64 * 1024 * 1024;
pile.Start();
var data = new PilePointer[cnt];
for (var i = 0; i < cnt; i++)
{
data[i] = pile.Put(new byte[0], preallocateBlockSize: Memory.PTR_RAW_BYTE_SIZE + (137 * (i % 17)));
}
var lst = new List <Task>();
long statRead = 0;
long statPut = 0;
long statDelete = 0;
var sw = Stopwatch.StartNew();
var sd = DateTime.UtcNow;
var deleteLock = new object();
for (var i = 0; i < tcount; i++)
{
lst.Add(Task.Factory.StartNew(() =>
{
while (true)
{
var now = DateTime.UtcNow;
if ((now - sd).TotalSeconds > seconds)
{
break;
}
var it = ExternalRandomGenerator.Instance.NextScaledRandomInteger(10, 100);
for (var j = 0; j < it; j++)
{
var pp = data[ExternalRandomGenerator.Instance.NextScaledRandomInteger(0, cnt - 1)];
Aver.IsTrue(pile.Get(pp) is byte[]);
Interlocked.Increment(ref statRead);
}
it = ExternalRandomGenerator.Instance.NextScaledRandomInteger(10, 100);
for (var j = 0; j < it; j++)
{
var pp = data[ExternalRandomGenerator.Instance.NextScaledRandomInteger(0, cnt - 1)];
Aver.IsTrue(pile.Put(pp, new byte[ExternalRandomGenerator.Instance.NextScaledRandomInteger(0, 3791)], link: true));
Interlocked.Increment(ref statPut);
}
if (ExternalRandomGenerator.Instance.NextScaledRandomInteger(0, 100) > 50 && Monitor.TryEnter(deleteLock))
{
try
{
var newData = (PilePointer[])data.Clone();
it = ExternalRandomGenerator.Instance.NextScaledRandomInteger(0, 10 + (cnt / 2));
var toDelete = new List <PilePointer>();
for (var j = 0; j < it; j++)
{
var idx = ExternalRandomGenerator.Instance.NextScaledRandomInteger(0, cnt - 1);
toDelete.Add(newData[idx]);
newData[idx] = pile.Put(new byte[12], preallocateBlockSize: ExternalRandomGenerator.Instance.NextScaledRandomInteger(24, 1024));
}
data = newData;//atomic;
Thread.Sleep(1000);
foreach (var pp in toDelete)
{
Aver.IsTrue(pile.Delete(pp));
Interlocked.Increment(ref statDelete);
}
}
finally
{
Monitor.Exit(deleteLock);
}
}
}
}, TaskCreationOptions.LongRunning));
}
Task.WaitAll(lst.ToArray());
var el = sw.ElapsedMilliseconds;
Console.WriteLine("Read {0:n0} at {1:n0} ops/sec".Args(statRead, statRead / (el / 1000d)));
Console.WriteLine("Put {0:n0} at {1:n0} ops/sec".Args(statPut, statPut / (el / 1000d)));
Console.WriteLine("Deleted {0:n0} at {1:n0} ops/sec".Args(statDelete, statDelete / (el / 1000d)));
for (var i = 0; i < data.Length; i++)
{
Aver.IsTrue(pile.Delete(data[i]));
}
Aver.AreEqual(0, pile.ObjectCount);
Aver.AreEqual(0, pile.ObjectLinkCount);
Aver.AreEqual(0, pile.AllocatedMemoryBytes);
}
}
public void VarSizes_Checkboard(bool isParallel, int cnt, int minSz, int maxSz, bool speed)
{
using (var pile = new DefaultPile())
{
pile.Start();
pile.AllocMode = speed ? AllocationMode.FavorSpeed : AllocationMode.ReuseSpace;
var tasks = new List <Task>();
for (var t = 0; t < (isParallel? (System.Environment.ProcessorCount - 1) : 1); t++)
{
tasks.Add(
Task.Run(() =>
{
var dict = new Dictionary <PilePointer, dummy>();
var priorOdd = PilePointer.Invalid;
for (var i = 0; i < cnt; i++)
{
var even = (i & 0x01) == 0;
var data = new dummy {
bin = new byte[12 + NFX.ExternalRandomGenerator.Instance.NextScaledRandomInteger(minSz, maxSz)]
};
data.bin.WriteBEInt32(0, ExternalRandomGenerator.Instance.NextRandomInteger);
data.bin.WriteBEInt32(data.bin.Length - 4, ExternalRandomGenerator.Instance.NextRandomInteger);
var ptr = pile.Put(data);
Assert.IsTrue(ptr.Valid);
if (even)
{
dict.Add(ptr, data);
}
else
{
if (priorOdd.Valid)
{
Assert.IsTrue(pile.Delete(priorOdd));
}
priorOdd = ptr;
}
if (i % 1000 == 0)
{
Console.WriteLine("Thread{0} did {1}; allocated {2} bytes, utilized {3} bytes by {4} objects {5} bytes/obj. ",
Thread.CurrentThread.ManagedThreadId,
i,
pile.AllocatedMemoryBytes,
pile.UtilizedBytes,
pile.ObjectCount,
pile.UtilizedBytes / pile.ObjectCount);
}
}
Console.WriteLine("Thread {0} Population done, now checking the buffers... {1}", Thread.CurrentThread.ManagedThreadId, DateTime.Now);
foreach (var entry in dict)
{
Assert.IsTrue(NFX.IOMiscUtils.MemBufferEquals(entry.Value.bin, (pile.Get(entry.Key) as dummy).bin));
}
Console.WriteLine("Thread {0} DONE. {1}", Thread.CurrentThread.ManagedThreadId, DateTime.Now);
})
);//add
}
Task.WaitAll(tasks.ToArray());
}
}
public void VarSizes_Increasing_Random(bool isParallel, int cnt, int minSz, int maxSz, bool speed, bool rnd)
{
using (var pile = new DefaultPile())
{
pile.Start();
pile.AllocMode = speed ? AllocationMode.FavorSpeed : AllocationMode.ReuseSpace;
var sw = Stopwatch.StartNew();
var tasks = new List <Task>();
for (var t = 0; t < (isParallel? (System.Environment.ProcessorCount - 1) : 1); t++)
{
tasks.Add(
Task.Run(() =>
{
var dict = new Dictionary <PilePointer, dummy>();
var lst = new List <PilePointer>();
var priorOdd = PilePointer.Invalid;
for (var i = 0; i < cnt; i++)
{
var buf = new byte[12 +
minSz +
(
rnd
? (NFX.ExternalRandomGenerator.Instance.NextScaledRandomInteger(0, (int)(maxSz * (i / (double)cnt))))
: (int)(maxSz * (i / (double)cnt))
)
];
var data = new dummy {
bin = buf
};
data.bin.WriteBEInt32(0, ExternalRandomGenerator.Instance.NextRandomInteger);
data.bin.WriteBEInt32(data.bin.Length - 4, ExternalRandomGenerator.Instance.NextRandomInteger);
var ptr = pile.Put(data);
Assert.IsTrue(ptr.Valid);
dict.Add(ptr, data);
lst.Add(ptr);
if (i > cnt / 3)
{
if (ExternalRandomGenerator.Instance.NextRandomInteger > 0)
{
var ri = ExternalRandomGenerator.Instance.NextScaledRandomInteger(0, lst.Count - 1);
var pp = lst[ri];
if (!pp.Valid)
{
continue;
}
Assert.IsTrue(pile.Delete(pp));
dict.Remove(pp);
lst[ri] = PilePointer.Invalid;
}
}
if (i % 1000 == 0)
{
Console.WriteLine("Thread{0} did {1}; allocated {2} bytes, utilized {3} bytes by {4} objects {5} bytes/obj. ",
Thread.CurrentThread.ManagedThreadId,
i,
pile.AllocatedMemoryBytes,
pile.UtilizedBytes,
pile.ObjectCount,
pile.UtilizedBytes / pile.ObjectCount);
}
}
Console.WriteLine("Thread {0} Population done, now checking the buffers... {1}", Thread.CurrentThread.ManagedThreadId, DateTime.Now);
foreach (var entry in dict)
{
Assert.IsTrue(NFX.IOMiscUtils.MemBufferEquals(entry.Value.bin, (pile.Get(entry.Key) as dummy).bin));
}
Console.WriteLine("Thread {0} DONE. {1}", Thread.CurrentThread.ManagedThreadId, DateTime.Now);
})
);//add
}
Task.WaitAll(tasks.ToArray());
var el = sw.ElapsedMilliseconds;
var gt = cnt * tasks.Count;
Console.WriteLine("Total objects: {0:n0} in {1:n0} ms at {2:n0} obj/sec".Args(gt, el, gt / (el / 1000d)));
}
}
public static void DeleteSeveral_TRow(bool speed, int durationSec, int putMin, int putMax, int delFactor, bool isParallel)
{
using (var pile = new DefaultPile())
{
pile.AllocMode = speed ? AllocationMode.FavorSpeed : AllocationMode.ReuseSpace;
pile.Start();
var startTime = DateTime.UtcNow;
var tasks = new List <Task>();
for (var t = 0; t < (isParallel ? (System.Environment.ProcessorCount - 1) : 1); t++)
{
tasks.Add(Task.Factory.StartNew(() =>
{
var list = new List <CheckTRow>();
var wlc = 0;
while (true)
{
if ((DateTime.UtcNow - startTime).TotalSeconds >= durationSec)
{
break;
}
var putCount = NFX.ExternalRandomGenerator.Instance.NextScaledRandomInteger(putMin, putMax);
for (int i = 0; i < putCount; i++)
{
var val = PersonRow.MakeFake(new GDID());
var ptr = pile.Put(val);
list.Add(new CheckTRow(ptr, val.ID, val.Address1));
}
// delete several random elements
int delCount = putCount / delFactor;
for (int i = 0; i < delCount; i++)
{
var idx = NFX.ExternalRandomGenerator.Instance.NextScaledRandomInteger(0, list.Count - 1);
var ptr = list[idx].Ptr;
pile.Delete(ptr);
list.RemoveAt(idx);
}
// get several random elements
if (list.Count > 64 && NFX.ExternalRandomGenerator.Instance.NextScaledRandomInteger(0, 100) > 98)
{
var toRead = NFX.ExternalRandomGenerator.Instance.NextScaledRandomInteger(8, 64);
wlc++;
if (wlc % 125 == 0)
{
Console.WriteLine("Thread {0} is reading {1} elements, total {2}"
.Args(Thread.CurrentThread.ManagedThreadId, toRead, list.Count));
}
for (var k = 0; k < toRead; k++)
{
var element = list[NFX.ExternalRandomGenerator.Instance.NextScaledRandomInteger(0, list.Count - 1)];
var buf = pile.Get(element.Ptr) as PersonRow;
Aver.IsTrue(element.Id.Equals(buf.ID));
Aver.IsTrue(element.Address.Equals(buf.Address1));
}
}
}
// total check
Console.WriteLine("Thread {0} is doing final read of {1} elements, objectCount {2}"
.Args(Thread.CurrentThread.ManagedThreadId, list.Count, pile.ObjectCount));
foreach (var element in list)
{
var buf = pile.Get(element.Ptr) as PersonRow;
Aver.IsTrue(element.Id.Equals(buf.ID));
Aver.IsTrue(element.Address.Equals(buf.Address1));
}
return;
}, TaskCreationOptions.LongRunning));
}
Task.WaitAll(tasks.ToArray());
}
}
public static void DeleteSeveral_ByteArray(bool speed, int durationSec, int putMin, int putMax, int delFactor, int payloadSizeMin, int payloadSizeMax, bool isParallel)
{
using (var pile = new DefaultPile())
{
pile.AllocMode = speed ? AllocationMode.FavorSpeed : AllocationMode.ReuseSpace;
pile.Start();
var startTime = DateTime.UtcNow;
var tasks = new List <Task>();
for (var t = 0; t < (isParallel ? (System.Environment.ProcessorCount - 1) : 1); t++)
{
tasks.Add(Task.Factory.StartNew(() =>
{
var list = new List <CheckByteArray>();
var wlc = 0;
while (true)
{
if ((DateTime.UtcNow - startTime).TotalSeconds >= durationSec)
{
break;
}
var putCount = NFX.ExternalRandomGenerator.Instance.NextScaledRandomInteger(putMin, putMax);
for (int i = 0; i < putCount; i++)
{
var payloadSize = NFX.ExternalRandomGenerator
.Instance.NextScaledRandomInteger(payloadSizeMin, payloadSizeMax);
var val = new byte[payloadSize];
val[0] = (byte)NFX.ExternalRandomGenerator.Instance.NextRandomInteger;
val[payloadSize - 1] = (byte)NFX.ExternalRandomGenerator.Instance.NextRandomInteger;
var ptr = pile.Put(val);
list.Add(new CheckByteArray(ptr, payloadSize - 1, val[0], val[payloadSize - 1]));
}
int delCount = putCount / delFactor;
for (int i = 0; i < delCount; i++)
{
var idx = NFX.ExternalRandomGenerator.Instance.NextScaledRandomInteger(0, list.Count - 1);
var ptr = list[idx].Ptr;
pile.Delete(ptr);
list.RemoveAt(idx);
}
// get several random elements
if (list.Count > 64 && NFX.ExternalRandomGenerator.Instance.NextScaledRandomInteger(0, 100) > 98)
{
var toRead = NFX.ExternalRandomGenerator.Instance.NextScaledRandomInteger(8, 64);
wlc++;
if (wlc % 125 == 0)
{
Console.WriteLine("Thread {0} is reading {1} elements, total {2}"
.Args(Thread.CurrentThread.ManagedThreadId, toRead, list.Count));
}
for (var k = 0; k < toRead; k++)
{
var element = list[NFX.ExternalRandomGenerator.Instance.NextScaledRandomInteger(0, list.Count - 1)];
var buf = pile.Get(element.Ptr) as byte[];
Aver.AreEqual(element.FirstByte, buf[0]);
Aver.AreEqual(element.LastByte, buf[element.IdxLast]);
}
}
if (list.Count == Int32.MaxValue)
{
list = new List <CheckByteArray>();
}
}
// total check
Console.WriteLine("Thread {0} is doing final read of {1} elements, objectCount {2}"
.Args(Thread.CurrentThread.ManagedThreadId, list.Count, pile.ObjectCount));
foreach (var element in list)
{
var buf = pile.Get(element.Ptr) as byte[];
Aver.AreEqual(element.FirstByte, buf[0]);
Aver.AreEqual(element.LastByte, buf[element.IdxLast]);
}
return;
}));
}
Task.WaitAll(tasks.ToArray());
}
}
public static void Put_RandomDelete_ByteArray(int cnt, int durationSec, bool speed, int payloadSizeMin, int payloadSizeMax, int deleteFreq, bool isParallel)
{
using (var pile = new DefaultPile())
{
pile.AllocMode = speed ? AllocationMode.FavorSpeed : AllocationMode.ReuseSpace;
pile.Start();
var startTime = DateTime.UtcNow;
var tasks = new List <Task>();
for (var t = 0; t < (isParallel ? (System.Environment.ProcessorCount - 1) : 1); t++)
{
tasks.Add(Task.Factory.StartNew(() =>
{
var wlc = 0;
while (true)
{
if ((DateTime.UtcNow - startTime).TotalSeconds >= durationSec)
{
return;
}
var dict = new Dictionary <int, CheckByteArray>();
Console.WriteLine("Starting a batch of {0}".Args(cnt));
for (int i = 0; i < cnt; i++)
{
var payloadSize = NFX.ExternalRandomGenerator.Instance.NextScaledRandomInteger(payloadSizeMin, payloadSizeMax);
var val = new byte[payloadSize];
val[0] = (byte)NFX.ExternalRandomGenerator.Instance.NextRandomInteger;
val[payloadSize - 1] = (byte)NFX.ExternalRandomGenerator.Instance.NextRandomInteger;
var ptr = pile.Put(val);
var element = new CheckByteArray(ptr, payloadSize - 1, val[0], val[payloadSize - 1]);
dict.Add(i, element);
if (dict.Count > 0 && i % deleteFreq == 0)
{
while (true)
{
var idx = i - NFX.ExternalRandomGenerator.Instance.NextScaledRandomInteger(0, i);
CheckByteArray stored;
if (dict.TryGetValue(idx, out stored))
{
ptr = stored.Ptr;
pile.Delete(ptr);
dict.Remove(idx);
break;
}
}
}
if (dict.Count > 16 && NFX.ExternalRandomGenerator.Instance.NextScaledRandomInteger(0, 100) > 98)
{
var toRead = NFX.ExternalRandomGenerator.Instance.NextScaledRandomInteger(8, 64);
wlc++;
if (wlc % 125 == 0)
{
Console.WriteLine("Thread {0} is reading {1} elements, total {2}"
.Args(Thread.CurrentThread.ManagedThreadId, toRead, dict.Count));
}
for (var k = 0; k < toRead; k++)
{
var kvp = dict.Skip(NFX.ExternalRandomGenerator.Instance.NextScaledRandomInteger(0, dict.Count - 1)).First();
var buf = pile.Get(kvp.Value.Ptr) as byte[];
Aver.AreEqual(kvp.Value.FirstByte, buf[0]);
Aver.AreEqual(kvp.Value.LastByte, buf[kvp.Value.IdxLast]);
}
}
}
Console.WriteLine("Thread {0} is doing final read of {1} elements".Args(Thread.CurrentThread.ManagedThreadId, dict.Count));
foreach (var kvp in dict)
{
var buf = pile.Get(kvp.Value.Ptr) as byte[];
Aver.AreEqual(kvp.Value.FirstByte, buf[0]);
Aver.AreEqual(kvp.Value.LastByte, buf[kvp.Value.IdxLast]);
}
}
}, TaskCreationOptions.LongRunning));
}
Task.WaitAll(tasks.ToArray());
}
}
public static void Chessboard_TRow(bool speed, int durationSec, bool isParallel)
{
using (var pile = new DefaultPile(NOPApplication.Instance))
{
pile.AllocMode = speed ? AllocationMode.FavorSpeed : AllocationMode.ReuseSpace;
pile.Start();
var startTime = DateTime.UtcNow;
var tasks = new List <Task>();
for (var t = 0; t < (isParallel ? (System.Environment.ProcessorCount - 1) : 1); t++)
{
tasks.Add(Task.Factory.StartNew(() =>
{
var list = new List <CheckTRow>();
var i = 0;
var wlc = 0;
while (true)
{
if ((DateTime.UtcNow - startTime).TotalSeconds >= durationSec)
{
break;
}
var val = PersonRow.MakeFake(new GDID(0, (ulong)i));
var ptr = pile.Put(val);
var element = new CheckTRow(ptr, val.ID, val.Address1);
list.Add(element);
// delete previous element
if (list.Count > 1 && i % 2 == 0)
{
ptr = list[list.Count - 2].Ptr;
pile.Delete(ptr);
list.RemoveAt(list.Count - 2);
}
// get several random elements
if (list.Count > 64 && Ambient.Random.NextScaledRandomInteger(0, 100) > 98)
{
var toRead = Ambient.Random.NextScaledRandomInteger(8, 64);
wlc++;
if (wlc % 125 == 0)
{
Console.WriteLine("Thread {0} is reading {1} elements, total {2}, Pile objects {3}, Pile segments {4} Pile Bytes {5}"
.Args(Thread.CurrentThread.ManagedThreadId, toRead, list.Count, pile.ObjectCount, pile.SegmentCount, pile.AllocatedMemoryBytes));
}
for (var k = 0; k < toRead; k++)
{
element = list[Ambient.Random.NextScaledRandomInteger(0, list.Count - 1)];
var buf = pile.Get(element.Ptr) as PersonRow;
Aver.IsTrue(element.Id.Equals(buf.ID));
Aver.IsTrue(element.Address.Equals(buf.Address1));
}
}
if (i == Int32.MaxValue)
{
i = 0;
}
else
{
i++;
}
if (list.Count == Int32.MaxValue)
{
list = new List <CheckTRow>();
}
}
// total check
Console.WriteLine("Thread {0} is doing final read of {1} elements, objectCount {2}"
.Args(Thread.CurrentThread.ManagedThreadId, list.Count, pile.ObjectCount));
foreach (var element in list)
{
var buf = pile.Get(element.Ptr) as PersonRow;
Aver.IsTrue(element.Id.Equals(buf.ID));
Aver.IsTrue(element.Address.Equals(buf.Address1));
}
return;
}, TaskCreationOptions.LongRunning));
}
Task.WaitAll(tasks.ToArray());
}
}
