public void ReallocateInPlace(int len)
{
using (var pile = new DefaultPile())
{
pile.Start();
var lst = new List <PilePointer>();
for (var i = 0; i < len; i++)
{
var ub = i * 10;
var ptr = pile.Put(new byte[0], preallocateBlockSize: ub);
lst.Add(ptr);
for (var j = 0; j < ub + (ub / 2); j++)
{
Aver.IsTrue(pile.Put(ptr, new byte[j]));
}
}
Aver.AreEqual(lst.Count, pile.ObjectCount);
var n = 0;
foreach (var pp in lst)
{
Console.WriteLine("iteration#{0} of {1}".Args(n, lst.Count));
Aver.IsTrue(pile.Delete(pp));
n++;
}
Aver.AreEqual(0, pile.AllocatedMemoryBytes);
Aver.AreEqual(0, pile.ObjectCount);
Aver.AreEqual(0, pile.ObjectLinkCount);
}
}
public void LinkStringNoPreallocate()
{
using (var pile = new DefaultPile(NOPApplication.Instance))
{
pile.Start();
var obj1 = "abcdefgh";
var p1 = pile.Put(obj1);
var got = pile.Get(p1) as string;
Aver.AreEqual("abcdefgh", got);
var obj2 = "abcdefghijklmnopqrst0912345";
Aver.IsTrue(pile.Put(p1, obj2));
Aver.AreEqual(1, pile.ObjectCount);
got = pile.Get(p1) as string;
Aver.AreEqual("abcdefghijklmnopqrst0912345", got);
Aver.IsTrue(pile.AllocatedMemoryBytes > 0);
Aver.IsTrue(pile.Delete(p1));
Aver.IsTrue(pile.AllocatedMemoryBytes == 0);
Aver.IsTrue(pile.ObjectCount == 0);
}
}
public void FitPreallocateString()
{
using (var pile = new DefaultPile())
{
pile.Start();
var obj1 = "abcdefgh";
var p1 = pile.Put(obj1, preallocateBlockSize: 4000);
var got = pile.Get(p1) as string;
Aver.AreEqual("abcdefgh", got);
Aver.AreEqual(4000, pile.SizeOf(p1));
var obj2 = "abcdefghijklmnopqrst0912345";
Aver.IsTrue(pile.Put(p1, obj2));
Aver.AreEqual(1, pile.ObjectCount);
got = pile.Get(p1) as string;
Aver.AreEqual("abcdefghijklmnopqrst0912345", got);
Aver.IsTrue(pile.AllocatedMemoryBytes > 0);
Aver.IsTrue(pile.Delete(p1));
Aver.IsTrue(pile.AllocatedMemoryBytes == 0);
Aver.IsTrue(pile.ObjectCount == 0);
}
}
[Run("cnt=32000000 size=32 parallel=12")] //12 threads put 32,000,000 byte[32] in 5,008 msec at 6,389,776 ops/sec
//12 threads got 32,000,000 byte[32] in 3,787 msec at 8,449,960 ops/sec
//--------------------------------------------------------------------------------
//12 threads put 32,000,000 byte[32] in 2,963 msec at 10,799,865 ops/sec
//12 threads got 32,000,000 byte[32] in 2,253 msec at 14,203,285 ops/sec
public void ByteBuf(int CNT, int size, int parallel)
{
using (var pile = new DefaultPile(NOPApplication.Instance))
{
pile.Start();
var str = new byte[size];
var pp = pile.Put(str);
var sw = Stopwatch.StartNew();
Parallel.For(0, CNT, new ParallelOptions {
MaxDegreeOfParallelism = parallel
},
(i) =>
{
pile.Put(str);
});
var el = sw.ElapsedMilliseconds;
Console.WriteLine("{0:n0} threads put {1:n0} byte[{2:n0}] in {3:n0} msec at {4:n0} ops/sec".Args(parallel, CNT, size, el, CNT / (el / 1000d)));
sw.Restart();
Parallel.For(0, CNT, new ParallelOptions {
MaxDegreeOfParallelism = parallel
},
(i) =>
{
pile.Get(pp);
});
el = sw.ElapsedMilliseconds;
Console.WriteLine("{0:n0} threads got {1:n0} byte[{2:n0}] in {3:n0} msec at {4:n0} ops/sec".Args(parallel, CNT, size, el, CNT / (el / 1000d)));
}
}
public void FitPreallocateByteArray()
{
using (var pile = new DefaultPile())
{
pile.Start();
var obj1 = new byte[3];
var p1 = pile.Put(obj1, preallocateBlockSize: 4000);
var got = pile.Get(p1) as byte[];
Aver.AreEqual(3, got.Length);
Aver.AreEqual(4000, pile.SizeOf(p1));
var obj2 = new byte[571];
Aver.IsTrue(pile.Put(p1, obj2));
Aver.AreEqual(1, pile.ObjectCount);
got = pile.Get(p1) as byte[];
Aver.AreEqual(571, got.Length);
Aver.IsTrue(pile.AllocatedMemoryBytes > 0);
Aver.IsTrue(pile.Delete(p1));
Aver.IsTrue(pile.AllocatedMemoryBytes == 0);
Aver.IsTrue(pile.ObjectCount == 0);
}
}
public void LinkPreallocate()
{
using (var pile = new DefaultPile(NOPApplication.Instance))
{
pile.Start();
var obj1 = new Payload {
ID = 1, Name = "1", Data = null
};
var p1 = pile.Put(obj1, preallocateBlockSize: 2000);
Aver.AreEqual(2000, pile.SizeOf(p1));
var obj2 = new Payload {
ID = 2, Name = "2", Data = new byte [3000]
};
Aver.IsTrue(pile.Put(p1, obj2));
Aver.AreEqual(1, pile.ObjectCount);
var got = pile.Get(p1) as Payload;
Aver.AreEqual(2, got.ID);
Aver.AreEqual("2", got.Name);
Aver.IsNotNull(got.Data);
Aver.AreEqual(3000, got.Data.Length);
Aver.IsTrue(pile.AllocatedMemoryBytes > 0);
Aver.IsTrue(pile.Delete(p1));
Aver.IsTrue(pile.AllocatedMemoryBytes == 0);
Aver.IsTrue(pile.ObjectCount == 0);
}
}
public void PutSubarray()
{
using (var pile = new DefaultPile(NOPApplication.Instance))
{
pile.Start();
var src = new byte[] { 1, 6, 9, 250 };
var sub = new Subarray <byte>();
sub.Set(src, 2);
var ptr1 = pile.Put(sub);
sub.Set(src, 4);
var ptr2 = pile.Put(sub);
var got1 = pile.Get(ptr1) as byte[];
var got2 = pile.Get(ptr2) as byte[];
Aver.IsNotNull(got1);
Aver.AreEqual(2, got1.Length);
Aver.AreEqual(1, got1[0]);
Aver.AreEqual(6, got1[1]);
Aver.IsNotNull(got2);
Aver.AreEqual(4, got2.Length);
Aver.AreEqual(1, got2[0]);
Aver.AreEqual(6, got2[1]);
Aver.AreEqual(9, got2[2]);
Aver.AreEqual(250, got2[3]);
}
}
public void LinkByteArrayNoPreallocate()
{
using (var pile = new DefaultPile())
{
pile.Start();
var obj1 = new byte[12];
var p1 = pile.Put(obj1);
var got = pile.Get(p1) as byte[];
Aver.AreEqual(12, got.Length);
var obj2 = new byte[389];
Aver.IsTrue(pile.Put(p1, obj2));
Aver.AreEqual(1, pile.ObjectCount);
got = pile.Get(p1) as byte[];
Aver.AreEqual(389, got.Length);
Aver.IsTrue(pile.AllocatedMemoryBytes > 0);
Aver.IsTrue(pile.Delete(p1));
Aver.IsTrue(pile.AllocatedMemoryBytes == 0);
Aver.IsTrue(pile.ObjectCount == 0);
}
}
public void LinkNoPreallocate()
{
using (var pile = new DefaultPile())
{
pile.Start();
var obj1 = new Payload {
ID = 1, Name = "1", Data = null
};
var p1 = pile.Put(obj1);
Aver.IsTrue(pile.SizeOf(p1) < 128);
var obj2 = new Payload {
ID = 2, Name = "2", Data = new byte [128]
};
Aver.IsTrue(pile.Put(p1, obj2));
Aver.AreEqual(1, pile.ObjectCount);
var got = pile.Get(p1) as Payload;
Aver.AreEqual(2, got.ID);
Aver.AreEqual("2", got.Name);
Aver.IsNotNull(got.Data);
Aver.AreEqual(128, got.Data.Length);
Aver.IsTrue(pile.AllocatedMemoryBytes > 0);
Aver.IsTrue(pile.Delete(p1));
Aver.IsTrue(pile.AllocatedMemoryBytes == 0);
Aver.IsTrue(pile.ObjectCount == 0);
}
}
public void FitPreallocate()
{
using (var pile = new DefaultPile())
{
pile.Start();
var obj1 = new Payload {
ID = 1, Name = "1", Data = null
};
var p1 = pile.Put(obj1, preallocateBlockSize: 4000);
Aver.AreEqual(4000, pile.SizeOf(p1));
var obj2 = new Payload {
ID = 2, Name = "2", Data = new byte [] { 1, 2, 3, 4, 5, 6, 7, 8 }
};
Aver.IsTrue(pile.Put(p1, obj2));
Aver.AreEqual(1, pile.ObjectCount);
var got = pile.Get(p1) as Payload;
Aver.AreEqual(2, got.ID);
Aver.AreEqual("2", got.Name);
Aver.IsNotNull(got.Data);
Aver.AreEqual(8, got.Data.Length);
Aver.IsTrue(pile.AllocatedMemoryBytes > 0);
Aver.IsTrue(pile.Delete(p1));
Aver.IsTrue(pile.AllocatedMemoryBytes == 0);
Aver.IsTrue(pile.ObjectCount == 0);
}
}
// private List<Person> m_Crap = new List<Person>();
private void btnPersonPut_Click(object sender, EventArgs e)
{
lbPerson.Items.Clear();
var cnt = tbPersonCount.Text.AsInt(10);
var w = Stopwatch.StartNew();
var pv = tbPersonVariance.Text.AsInt(0);
for (var i = 0; i < cnt; i++)
{
var obj = i % 5 == 0? Person2.MakeFake2() : Person.MakeFake();
//m_Crap.Add( obj );
if (pv > 0)
{
obj.BinData = new byte[pv];
}
var pp = m_Pile.Put(obj);
if (i < 25)
{
lbPerson.Items.Add(pp);
}
}
var elps = w.ElapsedMilliseconds;
Text = "Added {0:n0} in {1:n0}ms at {2:n0}/sec".Args(cnt, elps, cnt / (elps / 1000d));
}
public static void NoGrowth_ByteArray(bool speed, int durationSec, int payloadSizeMin, int payloadSizeMax, int countMin, int countMax)
{
using (var pile = new DefaultPile(NOPApplication.Instance))
{
pile.AllocMode = speed ? AllocationMode.FavorSpeed : AllocationMode.ReuseSpace;
pile.Start();
var startTime = DateTime.UtcNow;
var tasks = new List <Task>();
for (var t = 0; t < (System.Environment.ProcessorCount - 1); t++)
{
tasks.Add(Task.Factory.StartNew(() =>
{
var list = new List <CheckByteArray>();
bool put = true;
while (true)
{
if ((DateTime.UtcNow - startTime).TotalSeconds >= durationSec)
{
return;
}
if (put)
{
var cnt = Ambient.Random.NextScaledRandomInteger(countMin, countMax);
for (int j = 0; j < cnt; j++)
{
var payloadSize = Ambient.Random.NextScaledRandomInteger(payloadSizeMin, payloadSizeMax);
var val = new byte[payloadSize];
val[0] = (byte)Ambient.Random.NextRandomInteger;
val[payloadSize - 1] = (byte)Ambient.Random.NextRandomInteger;
var ptr = pile.Put(val);
var element = new CheckByteArray(ptr, payloadSize - 1, val[0], val[payloadSize - 1]);
list.Add(element);
}
Console.WriteLine("Thread {0} put {1} objects".Args(Thread.CurrentThread.ManagedThreadId, list.Count));
put = false;
}
else
{
Console.WriteLine("Thread {0} deleted {1} objects".Args(Thread.CurrentThread.ManagedThreadId, list.Count));
for (var j = 0; j < list.Count; j++)
{
var element = list[j];
var buf = pile.Get(element.Ptr) as byte[];
Aver.AreEqual(element.FirstByte, buf[0]);
Aver.AreEqual(element.LastByte, buf[element.IdxLast]);
pile.Delete(element.Ptr);
}
list.Clear();
put = true;
}
}
}, TaskCreationOptions.LongRunning));
}
Task.WaitAll(tasks.ToArray());
}
Console.WriteLine("Test finished.");
}
public void Buffers(int count, int segmentSize)
{
using (var pile = new DefaultPile())
{
pile.SegmentSize = segmentSize;
pile.Start();
var hs = new HashSet <int>();
for (var i = 0; i < count; i++)
{
var buf = new byte[4 + (i % 512)];
buf.WriteBEInt32(i);
pile.Put(buf);
hs.Add(i);
}
Console.WriteLine("Created {0} segments".Args(pile.SegmentCount));
var j = 0;
foreach (var entry in pile)
{
var buf = pile.Get(entry.Pointer) as byte[];
Aver.IsNotNull(buf);
Aver.IsTrue(buf.Length >= 4);
var i = buf.ReadBEInt32();
Aver.IsTrue(hs.Remove(i));
Aver.IsTrue(entry.Type == PileEntry.DataType.Buffer);
j++;
}
Aver.AreEqual(j, count);
Aver.AreEqual(0, hs.Count);
}
}
public void StringCorrectess(int len, int deleteEvery, int parallel)
{
using (var pile = new DefaultPile())
{
pile.Start();
var bag = new ConcurrentBag <PilePointer>();
var deleted = 0;
Parallel.For(0, len, new ParallelOptions {
MaxDegreeOfParallelism = parallel
},
(i) =>
{
var str = new string('a', i);
PilePointer pp = PilePointer.Invalid;
if (i % 2 == 0)
{
if (bag.TryTake(out pp))
{
pile.Put(pp, str);
}
}
pp = pile.Put(str, preallocateBlockSize: i + 100);
var got = pile.Get(pp) as string;
Aver.AreEqual(str, got);
if (i % deleteEvery == 0)
{
PilePointer dp;
if (bag.TryTake(out dp))
{
if (pile.Delete(dp, false))
{
Interlocked.Increment(ref deleted);
}
}
bag.Add(pp);
}
});
Console.WriteLine("Deleted {0:n0}", deleted);
}
}
public static void NoGrowth_TRow(bool speed, int durationSec, int countMin, int countMax)
{
using (var pile = new DefaultPile())
{
pile.AllocMode = speed ? AllocationMode.FavorSpeed : AllocationMode.ReuseSpace;
pile.Start();
var startTime = DateTime.UtcNow;
var tasks = new List <Task>();
for (var t = 0; t < (System.Environment.ProcessorCount - 1); t++)
{
tasks.Add(Task.Factory.StartNew(() =>
{
var list = new List <CheckTRow>();
bool put = true;
while (true)
{
if ((DateTime.UtcNow - startTime).TotalSeconds >= durationSec)
{
return;
}
if (put)
{
var cnt = NFX.ExternalRandomGenerator.Instance.NextScaledRandomInteger(countMin, countMax);
for (int j = 0; j < cnt; j++)
{
var val = PersonRow.MakeFake(new GDID());
var ptr = pile.Put(val);
var element = new CheckTRow(ptr, val.ID, val.Address1);
list.Add(element);
}
Console.WriteLine("Thread {0} put {1} objects".Args(Thread.CurrentThread.ManagedThreadId, list.Count));
put = false;
}
else
{
Console.WriteLine("Thread {0} deleted {1} objects".Args(Thread.CurrentThread.ManagedThreadId, list.Count));
for (var j = 0; j < list.Count; j++)
{
var element = list[j];
var buf = pile.Get(element.Ptr) as PersonRow;
Aver.AreEqual(element.Id, buf.ID);
Aver.AreEqual(element.Address, buf.Address1);
pile.Delete(element.Ptr);
}
list.Clear();
put = true;
}
}
}, TaskCreationOptions.LongRunning));
}
Task.WaitAll(tasks.ToArray());
}
Console.WriteLine("Test finished.");
}
public void MulticulturalString()
{
using (var pile = new DefaultPile(NOPApplication.Instance))
{
pile.Start();
var str = @"
外国語の学習と教授
Language Learning and Teaching
Изучение и обучение иностранных языков
Tere Daaheng Aneng Karimah
語文教學・语文教学
Enseñanza y estudio de idiomas
Изучаване и Преподаване на Чужди Езици
ქართული ენის შესწავლა და სწავლება
'læŋɡwidʒ 'lɘr:niŋ ænd 'ti:tʃiŋ
Lus kawm thaib qhia
Ngôn Ngữ, Sự học,
ללמוד וללמד את השֵפה
L'enseignement et l'étude des langues
말배우기와 가르치기
Nauka języków obcych
Γλωσσική Εκμὰθηση και Διδασκαλία
ﺗﺪﺭﯾﺲ ﻭ ﯾﺎﺩﮔﯿﺮﯼ ﺯﺑﺎﻥ
Sprachlernen und -lehren
ﺗﻌﻠﻢ ﻭﺗﺪﺭﻳﺲ ﺍﻟﻌﺮﺑﻴﺔ
เรียนและสอนภาษา";
var pp = pile.Put(str);
var got = pile.Get(pp) as string;
Aver.AreEqual(str, got);
}
}
public void PutString()
{
using (var pile = new DefaultPile(NOPApplication.Instance))
{
pile.Start();
var ptr = pile.Put("abcd123");
var got = pile.Get(ptr) as string;
Aver.IsNotNull(got);
Aver.AreEqual("abcd123", got);
}
}
public void Reallocate_Delete()
{
using (var pile = new DefaultPile())
{
pile.Start();
var obj1 = new Payload {
ID = 1, Name = "1", Data = null
};
var p1 = pile.Put(obj1, preallocateBlockSize: 2000);
Aver.AreEqual(2000, pile.SizeOf(p1));
var a = pile.AllocatedMemoryBytes;
var u = pile.UtilizedBytes;
var o = pile.OverheadBytes;
Console.WriteLine("Allocated: {0:n0} Utilized: {1:n0} Overhead: {2:n0}", a, u, o);
Aver.AreEqual(2000, u);
Aver.IsTrue(pile.Put(p1, new Payload {
Data = new byte[8000]
}));
var a2 = pile.AllocatedMemoryBytes;
var u2 = pile.UtilizedBytes;
var o2 = pile.OverheadBytes;
Console.WriteLine("Allocated: {0:n0} Utilized: {1:n0} Overhead: {2:n0}", a2, u2, o2);
Aver.IsTrue(u2 > 10000);
Aver.IsTrue(u2 > u);
pile.Delete(p1);
u2 = pile.UtilizedBytes;
Console.WriteLine("Allocated: {0:n0} Utilized: {1:n0} Overhead: {2:n0}", a2, u2, o2);
Aver.AreEqual(0, u2);
}
}
public void PutByteArray()
{
using (var pile = new DefaultPile(NOPApplication.Instance))
{
pile.Start();
var ptr = pile.Put(new byte[] { 1, 6, 9 });
var got = pile.Get(ptr) as byte[];
Aver.IsNotNull(got);
Aver.AreEqual(3, got.Length);
Aver.AreEqual(1, got[0]);
Aver.AreEqual(6, got[1]);
Aver.AreEqual(9, got[2]);
}
}
[Run("cnt=1048576")] // 1024 * 1024
public void LongMulticulturalString(int cnt)
{
using (var pile = new DefaultPile(NOPApplication.Instance))
{
pile.Start();
var str = new string('久', cnt);
var pp = pile.Put(str);
var got = pile.Get(pp) as string;
Aver.AreEqual(str, got);
}
}
public void SimpleTest()
{
using (var m_pile = new DefaultPile())
{
m_pile.Start();
var test = new LinkedListNode <int>(m_pile, 11);
var pp1 = m_pile.Put(11);
// var pp = m_pile.Put(test);
// Console.WriteLine(pp);
Console.WriteLine(test.Value);
Aver.AreEqual(test.Value, 11);
m_pile.WaitForCompleteStop();
}
}
public void EmptyTest()
{
using (var m_pile = new DefaultPile()
{
AllocMode = AllocationMode.FavorSpeed
})
{
m_pile.Start();
var test = m_pile.Put("TEST");
Console.WriteLine(test);
Console.WriteLine(m_pile.Get(test));
m_pile.WaitForCompleteStop();
}
Aver.IsTrue(true);
}
public void PutByteArray_GetDirectUnsafe()
{
using (var pile = new DefaultPile(NOPApplication.Instance))
{
pile.Start();
var ptr = pile.Put(new byte[] { 1, 6, 9 });
var got = pile.GetDirectMemoryBufferUnsafe(ptr, out var sflag);
Aver.IsNotNull(got.Array);
for (var i = 0; i < 8; i++)
{
"{0} {1:x2} {2}".SeeArgs(got.Offset, got.Array[got.Offset + i], (char)got.Array[got.Offset + i]);
}
Aver.AreEqual(3, got.Count);
Aver.AreEqual(1, got.Array[got.Offset + 0]);
Aver.AreEqual(6, got.Array[got.Offset + 1]);
Aver.AreEqual(9, got.Array[got.Offset + 2]);
}
}
public void ByteCorrectess(int len, int deleteEvery, int parallel)
{
using (var pile = new DefaultPile(NOPApplication.Instance))
{
pile.Start();
var bag = new ConcurrentBag <PilePointer>();
var deleted = 0;
Parallel.For(0, len, new ParallelOptions {
MaxDegreeOfParallelism = parallel
},
(i) =>
{
var original = new byte[len - i];
var pp = pile.Put(original);
var got = pile.Get(pp) as byte[];
Aver.AreEqual(original.Length, got.Length);
if (i % deleteEvery == 0)
{
PilePointer dp;
if (bag.TryTake(out dp))
{
if (pile.Delete(dp, false))
{
Interlocked.Increment(ref deleted);
}
}
bag.Add(pp);
}
});
Console.WriteLine("Deleted {0:n0}", deleted);
}
}
public void PileDeleteInMiddleSegment(int payloadSize)
{
using (var pile = new DefaultPile()
{
SegmentSize = PileCacheTestCore.SEG_SIZE
})
{
pile.Start();
var pps = new List <PilePointer>();
while (pile.SegmentCount < 4)
{
pps.Add(pile.Put(generatePayload(payloadSize)));
}
pile.Delete(pps.Last());
pps.RemoveAt(pps.Count - 1);
var objectsInsegmentCount = pps.Count / 3;
Console.WriteLine("{0:N0} object in pile, {1:N0} object per segment", pile.ObjectCount, objectsInsegmentCount);
Assert.AreEqual(3, pile.SegmentCount);
Assert.AreEqual(3, pile.SegmentTotalCount);
for (int i = objectsInsegmentCount; i < 2 * objectsInsegmentCount; i++)
{
pile.Delete(pps[i]);
}
Console.WriteLine("{0:N0} object in pile, {1:N0} segments, {2:N0} segments total", pile.ObjectCount, pile.SegmentCount, pile.SegmentTotalCount);
Assert.AreEqual(2, pile.SegmentCount);
Assert.AreEqual(3, pile.SegmentTotalCount);
}
}
public void PileDeleteInLastSegment(int payloadSize)
{
using (var pile = new DefaultPile()
{
SegmentSize = PileCacheTestCore.SEG_SIZE
})
{
pile.Start();
var pps = new List <PilePointer>();
while (pile.SegmentCount < 2)
{
pps.Add(pile.Put(generatePayload(payloadSize)));
}
pile.Delete(pps.Last());
pps.RemoveAt(pps.Count - 1);
Console.WriteLine("segment count: {0}, segment total count: {1}", pile.SegmentCount, pile.SegmentTotalCount);
Assert.AreEqual(1, pile.SegmentCount);
Assert.AreEqual(1, pile.SegmentTotalCount);
}
}
public static void DeleteSeveral_TRow(bool speed, int durationSec, int putMin, int putMax, int delFactor, bool isParallel)
{
using (var pile = new DefaultPile())
{
pile.AllocMode = speed ? AllocationMode.FavorSpeed : AllocationMode.ReuseSpace;
pile.Start();
var startTime = DateTime.UtcNow;
var tasks = new List <Task>();
for (var t = 0; t < (isParallel ? (System.Environment.ProcessorCount - 1) : 1); t++)
{
tasks.Add(Task.Factory.StartNew(() =>
{
var list = new List <CheckTRow>();
var wlc = 0;
while (true)
{
if ((DateTime.UtcNow - startTime).TotalSeconds >= durationSec)
{
break;
}
var putCount = NFX.ExternalRandomGenerator.Instance.NextScaledRandomInteger(putMin, putMax);
for (int i = 0; i < putCount; i++)
{
var val = PersonRow.MakeFake(new GDID());
var ptr = pile.Put(val);
list.Add(new CheckTRow(ptr, val.ID, val.Address1));
}
// delete several random elements
int delCount = putCount / delFactor;
for (int i = 0; i < delCount; i++)
{
var idx = NFX.ExternalRandomGenerator.Instance.NextScaledRandomInteger(0, list.Count - 1);
var ptr = list[idx].Ptr;
pile.Delete(ptr);
list.RemoveAt(idx);
}
// get several random elements
if (list.Count > 64 && NFX.ExternalRandomGenerator.Instance.NextScaledRandomInteger(0, 100) > 98)
{
var toRead = NFX.ExternalRandomGenerator.Instance.NextScaledRandomInteger(8, 64);
wlc++;
if (wlc % 125 == 0)
{
Console.WriteLine("Thread {0} is reading {1} elements, total {2}"
.Args(Thread.CurrentThread.ManagedThreadId, toRead, list.Count));
}
for (var k = 0; k < toRead; k++)
{
var element = list[NFX.ExternalRandomGenerator.Instance.NextScaledRandomInteger(0, list.Count - 1)];
var buf = pile.Get(element.Ptr) as PersonRow;
Aver.IsTrue(element.Id.Equals(buf.ID));
Aver.IsTrue(element.Address.Equals(buf.Address1));
}
}
}
// total check
Console.WriteLine("Thread {0} is doing final read of {1} elements, objectCount {2}"
.Args(Thread.CurrentThread.ManagedThreadId, list.Count, pile.ObjectCount));
foreach (var element in list)
{
var buf = pile.Get(element.Ptr) as PersonRow;
Aver.IsTrue(element.Id.Equals(buf.ID));
Aver.IsTrue(element.Address.Equals(buf.Address1));
}
return;
}, TaskCreationOptions.LongRunning));
}
Task.WaitAll(tasks.ToArray());
}
}
public static void DeleteSeveral_ByteArray(bool speed, int durationSec, int putMin, int putMax, int delFactor, int payloadSizeMin, int payloadSizeMax, bool isParallel)
{
using (var pile = new DefaultPile())
{
pile.AllocMode = speed ? AllocationMode.FavorSpeed : AllocationMode.ReuseSpace;
pile.Start();
var startTime = DateTime.UtcNow;
var tasks = new List <Task>();
for (var t = 0; t < (isParallel ? (System.Environment.ProcessorCount - 1) : 1); t++)
{
tasks.Add(Task.Factory.StartNew(() =>
{
var list = new List <CheckByteArray>();
var wlc = 0;
while (true)
{
if ((DateTime.UtcNow - startTime).TotalSeconds >= durationSec)
{
break;
}
var putCount = NFX.ExternalRandomGenerator.Instance.NextScaledRandomInteger(putMin, putMax);
for (int i = 0; i < putCount; i++)
{
var payloadSize = NFX.ExternalRandomGenerator
.Instance.NextScaledRandomInteger(payloadSizeMin, payloadSizeMax);
var val = new byte[payloadSize];
val[0] = (byte)NFX.ExternalRandomGenerator.Instance.NextRandomInteger;
val[payloadSize - 1] = (byte)NFX.ExternalRandomGenerator.Instance.NextRandomInteger;
var ptr = pile.Put(val);
list.Add(new CheckByteArray(ptr, payloadSize - 1, val[0], val[payloadSize - 1]));
}
int delCount = putCount / delFactor;
for (int i = 0; i < delCount; i++)
{
var idx = NFX.ExternalRandomGenerator.Instance.NextScaledRandomInteger(0, list.Count - 1);
var ptr = list[idx].Ptr;
pile.Delete(ptr);
list.RemoveAt(idx);
}
// get several random elements
if (list.Count > 64 && NFX.ExternalRandomGenerator.Instance.NextScaledRandomInteger(0, 100) > 98)
{
var toRead = NFX.ExternalRandomGenerator.Instance.NextScaledRandomInteger(8, 64);
wlc++;
if (wlc % 125 == 0)
{
Console.WriteLine("Thread {0} is reading {1} elements, total {2}"
.Args(Thread.CurrentThread.ManagedThreadId, toRead, list.Count));
}
for (var k = 0; k < toRead; k++)
{
var element = list[NFX.ExternalRandomGenerator.Instance.NextScaledRandomInteger(0, list.Count - 1)];
var buf = pile.Get(element.Ptr) as byte[];
Aver.AreEqual(element.FirstByte, buf[0]);
Aver.AreEqual(element.LastByte, buf[element.IdxLast]);
}
}
if (list.Count == Int32.MaxValue)
{
list = new List <CheckByteArray>();
}
}
// total check
Console.WriteLine("Thread {0} is doing final read of {1} elements, objectCount {2}"
.Args(Thread.CurrentThread.ManagedThreadId, list.Count, pile.ObjectCount));
foreach (var element in list)
{
var buf = pile.Get(element.Ptr) as byte[];
Aver.AreEqual(element.FirstByte, buf[0]);
Aver.AreEqual(element.LastByte, buf[element.IdxLast]);
}
return;
}));
}
Task.WaitAll(tasks.ToArray());
}
}
public void PileSmallObjects(int payloadSize, params int[] freeChunkSizes)
{
using (var pile = new DefaultPile()
{
SegmentSize = PileCacheTestCore.SEG_SIZE, AllocMode = AllocationMode.ReuseSpace
})
{
pile.FreeChunkSizes = freeChunkSizes;
pile.Start();
var pps = new List <PilePointer>();
while (pile.SegmentCount < 2)
{
pps.Add(pile.Put(generatePayload(payloadSize)));
}
pile.Delete(pps.Last());
pps.RemoveAt(pps.Count - 1);
Console.WriteLine("just removed the last added payload and segment should be 1 now, real segment count is {0}", pile.SegmentCount);
var objectsInFirstSegment = pps.Count;
Console.WriteLine("put {0:N0} objects in first segment, pile.ObjectCount {1:N0}", objectsInFirstSegment, pile.ObjectCount);
var deletedObjectCount = 0;
for (int i = 0; i < pps.Count; i += 2, deletedObjectCount++)
{
Assert.IsTrue(pile.Delete(pps[i]));
}
Console.WriteLine("deleted {0:N0} objects, pile.ObjectCount {1:N0}", deletedObjectCount, pile.ObjectCount);
var crawlStatus = pile.Crawl(false);
Console.WriteLine("crawl: {0}", crawlStatus);
var pps1 = new List <PilePointer>();
var c = 0;
while (pile.SegmentCount < 3)
{
pps1.Add(pile.Put(generatePayload(payloadSize)));
if (c % 20000 == 0)
{
pile.Crawl(true); //we do crawl because otherwise the 25000 free index xlots get exhausted AND
}
c++; //this unit tests does not run long enough to cause Crawl within allocator (5+ seconds)
} //so we induce Crawl by hand to rebiild indexes
pile.Delete(pps1.Last());
pps1.RemoveAt(pps1.Count - 1);
Console.WriteLine("again just removed the last added payload and segment should be 2 now, real segment count is {0}", pile.SegmentCount);
var objectsInSecondSegment = pps1.Count;
Console.WriteLine("put {0:N0} objects in second segment, pile.ObjectCount {1:N0}", objectsInSecondSegment, pile.ObjectCount);
Assert.AreEqual(objectsInFirstSegment * 2, pile.ObjectCount, "Object count in pile with two full segments must be equal to double object count in full first segment!");
}
}
public void TestThreadSafety(int cnt, int tcount, int seconds)
{
using (var pile = new DefaultPile())
{
// pile.SegmentSize = 64 * 1024 * 1024;
pile.Start();
var data = new PilePointer[cnt];
for (var i = 0; i < cnt; i++)
{
data[i] = pile.Put(new byte[0], preallocateBlockSize: Memory.PTR_RAW_BYTE_SIZE + (137 * (i % 17)));
}
var lst = new List <Task>();
long statRead = 0;
long statPut = 0;
long statDelete = 0;
var sw = Stopwatch.StartNew();
var sd = DateTime.UtcNow;
var deleteLock = new object();
for (var i = 0; i < tcount; i++)
{
lst.Add(Task.Factory.StartNew(() =>
{
while (true)
{
var now = DateTime.UtcNow;
if ((now - sd).TotalSeconds > seconds)
{
break;
}
var it = ExternalRandomGenerator.Instance.NextScaledRandomInteger(10, 100);
for (var j = 0; j < it; j++)
{
var pp = data[ExternalRandomGenerator.Instance.NextScaledRandomInteger(0, cnt - 1)];
Aver.IsTrue(pile.Get(pp) is byte[]);
Interlocked.Increment(ref statRead);
}
it = ExternalRandomGenerator.Instance.NextScaledRandomInteger(10, 100);
for (var j = 0; j < it; j++)
{
var pp = data[ExternalRandomGenerator.Instance.NextScaledRandomInteger(0, cnt - 1)];
Aver.IsTrue(pile.Put(pp, new byte[ExternalRandomGenerator.Instance.NextScaledRandomInteger(0, 3791)], link: true));
Interlocked.Increment(ref statPut);
}
if (ExternalRandomGenerator.Instance.NextScaledRandomInteger(0, 100) > 50 && Monitor.TryEnter(deleteLock))
{
try
{
var newData = (PilePointer[])data.Clone();
it = ExternalRandomGenerator.Instance.NextScaledRandomInteger(0, 10 + (cnt / 2));
var toDelete = new List <PilePointer>();
for (var j = 0; j < it; j++)
{
var idx = ExternalRandomGenerator.Instance.NextScaledRandomInteger(0, cnt - 1);
toDelete.Add(newData[idx]);
newData[idx] = pile.Put(new byte[12], preallocateBlockSize: ExternalRandomGenerator.Instance.NextScaledRandomInteger(24, 1024));
}
data = newData;//atomic;
Thread.Sleep(1000);
foreach (var pp in toDelete)
{
Aver.IsTrue(pile.Delete(pp));
Interlocked.Increment(ref statDelete);
}
}
finally
{
Monitor.Exit(deleteLock);
}
}
}
}, TaskCreationOptions.LongRunning));
}
Task.WaitAll(lst.ToArray());
var el = sw.ElapsedMilliseconds;
Console.WriteLine("Read {0:n0} at {1:n0} ops/sec".Args(statRead, statRead / (el / 1000d)));
Console.WriteLine("Put {0:n0} at {1:n0} ops/sec".Args(statPut, statPut / (el / 1000d)));
Console.WriteLine("Deleted {0:n0} at {1:n0} ops/sec".Args(statDelete, statDelete / (el / 1000d)));
for (var i = 0; i < data.Length; i++)
{
Aver.IsTrue(pile.Delete(data[i]));
}
Aver.AreEqual(0, pile.ObjectCount);
Aver.AreEqual(0, pile.ObjectLinkCount);
Aver.AreEqual(0, pile.AllocatedMemoryBytes);
}
}