private void btnPersonDelete_Click(object sender, EventArgs e) { if (lbPerson.SelectedItem == null) { return; } var pp = (PilePointer)lbPerson.SelectedItem; m_Pile.Delete(pp); lbPerson.Items.Remove(pp); }
public static void NoGrowth_ByteArray(bool speed, int durationSec, int payloadSizeMin, int payloadSizeMax, int countMin, int countMax) { using (var pile = new DefaultPile(NOPApplication.Instance)) { pile.AllocMode = speed ? AllocationMode.FavorSpeed : AllocationMode.ReuseSpace; pile.Start(); var startTime = DateTime.UtcNow; var tasks = new List <Task>(); for (var t = 0; t < (System.Environment.ProcessorCount - 1); t++) { tasks.Add(Task.Factory.StartNew(() => { var list = new List <CheckByteArray>(); bool put = true; while (true) { if ((DateTime.UtcNow - startTime).TotalSeconds >= durationSec) { return; } if (put) { var cnt = Ambient.Random.NextScaledRandomInteger(countMin, countMax); for (int j = 0; j < cnt; j++) { var payloadSize = Ambient.Random.NextScaledRandomInteger(payloadSizeMin, payloadSizeMax); var val = new byte[payloadSize]; val[0] = (byte)Ambient.Random.NextRandomInteger; val[payloadSize - 1] = (byte)Ambient.Random.NextRandomInteger; var ptr = pile.Put(val); var element = new CheckByteArray(ptr, payloadSize - 1, val[0], val[payloadSize - 1]); list.Add(element); } Console.WriteLine("Thread {0} put {1} objects".Args(Thread.CurrentThread.ManagedThreadId, list.Count)); put = false; } else { Console.WriteLine("Thread {0} deleted {1} objects".Args(Thread.CurrentThread.ManagedThreadId, list.Count)); for (var j = 0; j < list.Count; j++) { var element = list[j]; var buf = pile.Get(element.Ptr) as byte[]; Aver.AreEqual(element.FirstByte, buf[0]); Aver.AreEqual(element.LastByte, buf[element.IdxLast]); pile.Delete(element.Ptr); } list.Clear(); put = true; } } }, TaskCreationOptions.LongRunning)); } Task.WaitAll(tasks.ToArray()); } Console.WriteLine("Test finished."); }
public void FitPreallocateByteArray() { using (var pile = new DefaultPile()) { pile.Start(); var obj1 = new byte[3]; var p1 = pile.Put(obj1, preallocateBlockSize: 4000); var got = pile.Get(p1) as byte[]; Aver.AreEqual(3, got.Length); Aver.AreEqual(4000, pile.SizeOf(p1)); var obj2 = new byte[571]; Aver.IsTrue(pile.Put(p1, obj2)); Aver.AreEqual(1, pile.ObjectCount); got = pile.Get(p1) as byte[]; Aver.AreEqual(571, got.Length); Aver.IsTrue(pile.AllocatedMemoryBytes > 0); Aver.IsTrue(pile.Delete(p1)); Aver.IsTrue(pile.AllocatedMemoryBytes == 0); Aver.IsTrue(pile.ObjectCount == 0); } }
public void LinkNoPreallocate() { using (var pile = new DefaultPile()) { pile.Start(); var obj1 = new Payload { ID = 1, Name = "1", Data = null }; var p1 = pile.Put(obj1); Aver.IsTrue(pile.SizeOf(p1) < 128); var obj2 = new Payload { ID = 2, Name = "2", Data = new byte [128] }; Aver.IsTrue(pile.Put(p1, obj2)); Aver.AreEqual(1, pile.ObjectCount); var got = pile.Get(p1) as Payload; Aver.AreEqual(2, got.ID); Aver.AreEqual("2", got.Name); Aver.IsNotNull(got.Data); Aver.AreEqual(128, got.Data.Length); Aver.IsTrue(pile.AllocatedMemoryBytes > 0); Aver.IsTrue(pile.Delete(p1)); Aver.IsTrue(pile.AllocatedMemoryBytes == 0); Aver.IsTrue(pile.ObjectCount == 0); } }
public void LinkByteArrayNoPreallocate() { using (var pile = new DefaultPile()) { pile.Start(); var obj1 = new byte[12]; var p1 = pile.Put(obj1); var got = pile.Get(p1) as byte[]; Aver.AreEqual(12, got.Length); var obj2 = new byte[389]; Aver.IsTrue(pile.Put(p1, obj2)); Aver.AreEqual(1, pile.ObjectCount); got = pile.Get(p1) as byte[]; Aver.AreEqual(389, got.Length); Aver.IsTrue(pile.AllocatedMemoryBytes > 0); Aver.IsTrue(pile.Delete(p1)); Aver.IsTrue(pile.AllocatedMemoryBytes == 0); Aver.IsTrue(pile.ObjectCount == 0); } }
public void ReallocateInPlace(int len) { using (var pile = new DefaultPile()) { pile.Start(); var lst = new List <PilePointer>(); for (var i = 0; i < len; i++) { var ub = i * 10; var ptr = pile.Put(new byte[0], preallocateBlockSize: ub); lst.Add(ptr); for (var j = 0; j < ub + (ub / 2); j++) { Aver.IsTrue(pile.Put(ptr, new byte[j])); } } Aver.AreEqual(lst.Count, pile.ObjectCount); var n = 0; foreach (var pp in lst) { Console.WriteLine("iteration#{0} of {1}".Args(n, lst.Count)); Aver.IsTrue(pile.Delete(pp)); n++; } Aver.AreEqual(0, pile.AllocatedMemoryBytes); Aver.AreEqual(0, pile.ObjectCount); Aver.AreEqual(0, pile.ObjectLinkCount); } }
public void FitPreallocate() { using (var pile = new DefaultPile()) { pile.Start(); var obj1 = new Payload { ID = 1, Name = "1", Data = null }; var p1 = pile.Put(obj1, preallocateBlockSize: 4000); Aver.AreEqual(4000, pile.SizeOf(p1)); var obj2 = new Payload { ID = 2, Name = "2", Data = new byte [] { 1, 2, 3, 4, 5, 6, 7, 8 } }; Aver.IsTrue(pile.Put(p1, obj2)); Aver.AreEqual(1, pile.ObjectCount); var got = pile.Get(p1) as Payload; Aver.AreEqual(2, got.ID); Aver.AreEqual("2", got.Name); Aver.IsNotNull(got.Data); Aver.AreEqual(8, got.Data.Length); Aver.IsTrue(pile.AllocatedMemoryBytes > 0); Aver.IsTrue(pile.Delete(p1)); Aver.IsTrue(pile.AllocatedMemoryBytes == 0); Aver.IsTrue(pile.ObjectCount == 0); } }
public void FitPreallocateString() { using (var pile = new DefaultPile()) { pile.Start(); var obj1 = "abcdefgh"; var p1 = pile.Put(obj1, preallocateBlockSize: 4000); var got = pile.Get(p1) as string; Aver.AreEqual("abcdefgh", got); Aver.AreEqual(4000, pile.SizeOf(p1)); var obj2 = "abcdefghijklmnopqrst0912345"; Aver.IsTrue(pile.Put(p1, obj2)); Aver.AreEqual(1, pile.ObjectCount); got = pile.Get(p1) as string; Aver.AreEqual("abcdefghijklmnopqrst0912345", got); Aver.IsTrue(pile.AllocatedMemoryBytes > 0); Aver.IsTrue(pile.Delete(p1)); Aver.IsTrue(pile.AllocatedMemoryBytes == 0); Aver.IsTrue(pile.ObjectCount == 0); } }
public void LinkPreallocate() { using (var pile = new DefaultPile(NOPApplication.Instance)) { pile.Start(); var obj1 = new Payload { ID = 1, Name = "1", Data = null }; var p1 = pile.Put(obj1, preallocateBlockSize: 2000); Aver.AreEqual(2000, pile.SizeOf(p1)); var obj2 = new Payload { ID = 2, Name = "2", Data = new byte [3000] }; Aver.IsTrue(pile.Put(p1, obj2)); Aver.AreEqual(1, pile.ObjectCount); var got = pile.Get(p1) as Payload; Aver.AreEqual(2, got.ID); Aver.AreEqual("2", got.Name); Aver.IsNotNull(got.Data); Aver.AreEqual(3000, got.Data.Length); Aver.IsTrue(pile.AllocatedMemoryBytes > 0); Aver.IsTrue(pile.Delete(p1)); Aver.IsTrue(pile.AllocatedMemoryBytes == 0); Aver.IsTrue(pile.ObjectCount == 0); } }
public void LinkStringNoPreallocate() { using (var pile = new DefaultPile(NOPApplication.Instance)) { pile.Start(); var obj1 = "abcdefgh"; var p1 = pile.Put(obj1); var got = pile.Get(p1) as string; Aver.AreEqual("abcdefgh", got); var obj2 = "abcdefghijklmnopqrst0912345"; Aver.IsTrue(pile.Put(p1, obj2)); Aver.AreEqual(1, pile.ObjectCount); got = pile.Get(p1) as string; Aver.AreEqual("abcdefghijklmnopqrst0912345", got); Aver.IsTrue(pile.AllocatedMemoryBytes > 0); Aver.IsTrue(pile.Delete(p1)); Aver.IsTrue(pile.AllocatedMemoryBytes == 0); Aver.IsTrue(pile.ObjectCount == 0); } }
public static void NoGrowth_TRow(bool speed, int durationSec, int countMin, int countMax) { using (var pile = new DefaultPile()) { pile.AllocMode = speed ? AllocationMode.FavorSpeed : AllocationMode.ReuseSpace; pile.Start(); var startTime = DateTime.UtcNow; var tasks = new List <Task>(); for (var t = 0; t < (System.Environment.ProcessorCount - 1); t++) { tasks.Add(Task.Factory.StartNew(() => { var list = new List <CheckTRow>(); bool put = true; while (true) { if ((DateTime.UtcNow - startTime).TotalSeconds >= durationSec) { return; } if (put) { var cnt = NFX.ExternalRandomGenerator.Instance.NextScaledRandomInteger(countMin, countMax); for (int j = 0; j < cnt; j++) { var val = PersonRow.MakeFake(new GDID()); var ptr = pile.Put(val); var element = new CheckTRow(ptr, val.ID, val.Address1); list.Add(element); } Console.WriteLine("Thread {0} put {1} objects".Args(Thread.CurrentThread.ManagedThreadId, list.Count)); put = false; } else { Console.WriteLine("Thread {0} deleted {1} objects".Args(Thread.CurrentThread.ManagedThreadId, list.Count)); for (var j = 0; j < list.Count; j++) { var element = list[j]; var buf = pile.Get(element.Ptr) as PersonRow; Aver.AreEqual(element.Id, buf.ID); Aver.AreEqual(element.Address, buf.Address1); pile.Delete(element.Ptr); } list.Clear(); put = true; } } }, TaskCreationOptions.LongRunning)); } Task.WaitAll(tasks.ToArray()); } Console.WriteLine("Test finished."); }
public void PileDeleteInMiddleSegment(int payloadSize) { using (var pile = new DefaultPile() { SegmentSize = PileCacheTestCore.SEG_SIZE }) { pile.Start(); var pps = new List <PilePointer>(); while (pile.SegmentCount < 4) { pps.Add(pile.Put(generatePayload(payloadSize))); } pile.Delete(pps.Last()); pps.RemoveAt(pps.Count - 1); var objectsInsegmentCount = pps.Count / 3; Console.WriteLine("{0:N0} object in pile, {1:N0} object per segment", pile.ObjectCount, objectsInsegmentCount); Assert.AreEqual(3, pile.SegmentCount); Assert.AreEqual(3, pile.SegmentTotalCount); for (int i = objectsInsegmentCount; i < 2 * objectsInsegmentCount; i++) { pile.Delete(pps[i]); } Console.WriteLine("{0:N0} object in pile, {1:N0} segments, {2:N0} segments total", pile.ObjectCount, pile.SegmentCount, pile.SegmentTotalCount); Assert.AreEqual(2, pile.SegmentCount); Assert.AreEqual(3, pile.SegmentTotalCount); } }
public void StringCorrectess(int len, int deleteEvery, int parallel) { using (var pile = new DefaultPile()) { pile.Start(); var bag = new ConcurrentBag <PilePointer>(); var deleted = 0; Parallel.For(0, len, new ParallelOptions { MaxDegreeOfParallelism = parallel }, (i) => { var str = new string('a', i); PilePointer pp = PilePointer.Invalid; if (i % 2 == 0) { if (bag.TryTake(out pp)) { pile.Put(pp, str); } } pp = pile.Put(str, preallocateBlockSize: i + 100); var got = pile.Get(pp) as string; Aver.AreEqual(str, got); if (i % deleteEvery == 0) { PilePointer dp; if (bag.TryTake(out dp)) { if (pile.Delete(dp, false)) { Interlocked.Increment(ref deleted); } } bag.Add(pp); } }); Console.WriteLine("Deleted {0:n0}", deleted); } }
public void ByteCorrectess(int len, int deleteEvery, int parallel) { using (var pile = new DefaultPile(NOPApplication.Instance)) { pile.Start(); var bag = new ConcurrentBag <PilePointer>(); var deleted = 0; Parallel.For(0, len, new ParallelOptions { MaxDegreeOfParallelism = parallel }, (i) => { var original = new byte[len - i]; var pp = pile.Put(original); var got = pile.Get(pp) as byte[]; Aver.AreEqual(original.Length, got.Length); if (i % deleteEvery == 0) { PilePointer dp; if (bag.TryTake(out dp)) { if (pile.Delete(dp, false)) { Interlocked.Increment(ref deleted); } } bag.Add(pp); } }); Console.WriteLine("Deleted {0:n0}", deleted); } }
public void Reallocate_Delete() { using (var pile = new DefaultPile()) { pile.Start(); var obj1 = new Payload { ID = 1, Name = "1", Data = null }; var p1 = pile.Put(obj1, preallocateBlockSize: 2000); Aver.AreEqual(2000, pile.SizeOf(p1)); var a = pile.AllocatedMemoryBytes; var u = pile.UtilizedBytes; var o = pile.OverheadBytes; Console.WriteLine("Allocated: {0:n0} Utilized: {1:n0} Overhead: {2:n0}", a, u, o); Aver.AreEqual(2000, u); Aver.IsTrue(pile.Put(p1, new Payload { Data = new byte[8000] })); var a2 = pile.AllocatedMemoryBytes; var u2 = pile.UtilizedBytes; var o2 = pile.OverheadBytes; Console.WriteLine("Allocated: {0:n0} Utilized: {1:n0} Overhead: {2:n0}", a2, u2, o2); Aver.IsTrue(u2 > 10000); Aver.IsTrue(u2 > u); pile.Delete(p1); u2 = pile.UtilizedBytes; Console.WriteLine("Allocated: {0:n0} Utilized: {1:n0} Overhead: {2:n0}", a2, u2, o2); Aver.AreEqual(0, u2); } }
public void PileDeleteInLastSegment(int payloadSize) { using (var pile = new DefaultPile() { SegmentSize = PileCacheTestCore.SEG_SIZE }) { pile.Start(); var pps = new List <PilePointer>(); while (pile.SegmentCount < 2) { pps.Add(pile.Put(generatePayload(payloadSize))); } pile.Delete(pps.Last()); pps.RemoveAt(pps.Count - 1); Console.WriteLine("segment count: {0}, segment total count: {1}", pile.SegmentCount, pile.SegmentTotalCount); Assert.AreEqual(1, pile.SegmentCount); Assert.AreEqual(1, pile.SegmentTotalCount); } }
public static void DeleteSeveral_TRow(bool speed, int durationSec, int putMin, int putMax, int delFactor, bool isParallel) { using (var pile = new DefaultPile()) { pile.AllocMode = speed ? AllocationMode.FavorSpeed : AllocationMode.ReuseSpace; pile.Start(); var startTime = DateTime.UtcNow; var tasks = new List <Task>(); for (var t = 0; t < (isParallel ? (System.Environment.ProcessorCount - 1) : 1); t++) { tasks.Add(Task.Factory.StartNew(() => { var list = new List <CheckTRow>(); var wlc = 0; while (true) { if ((DateTime.UtcNow - startTime).TotalSeconds >= durationSec) { break; } var putCount = NFX.ExternalRandomGenerator.Instance.NextScaledRandomInteger(putMin, putMax); for (int i = 0; i < putCount; i++) { var val = PersonRow.MakeFake(new GDID()); var ptr = pile.Put(val); list.Add(new CheckTRow(ptr, val.ID, val.Address1)); } // delete several random elements int delCount = putCount / delFactor; for (int i = 0; i < delCount; i++) { var idx = NFX.ExternalRandomGenerator.Instance.NextScaledRandomInteger(0, list.Count - 1); var ptr = list[idx].Ptr; pile.Delete(ptr); list.RemoveAt(idx); } // get several random elements if (list.Count > 64 && NFX.ExternalRandomGenerator.Instance.NextScaledRandomInteger(0, 100) > 98) { var toRead = NFX.ExternalRandomGenerator.Instance.NextScaledRandomInteger(8, 64); wlc++; if (wlc % 125 == 0) { Console.WriteLine("Thread {0} is reading {1} elements, total {2}" .Args(Thread.CurrentThread.ManagedThreadId, toRead, list.Count)); } for (var k = 0; k < toRead; k++) { var element = list[NFX.ExternalRandomGenerator.Instance.NextScaledRandomInteger(0, list.Count - 1)]; var buf = pile.Get(element.Ptr) as PersonRow; Aver.IsTrue(element.Id.Equals(buf.ID)); Aver.IsTrue(element.Address.Equals(buf.Address1)); } } } // total check Console.WriteLine("Thread {0} is doing final read of {1} elements, objectCount {2}" .Args(Thread.CurrentThread.ManagedThreadId, list.Count, pile.ObjectCount)); foreach (var element in list) { var buf = pile.Get(element.Ptr) as PersonRow; Aver.IsTrue(element.Id.Equals(buf.ID)); Aver.IsTrue(element.Address.Equals(buf.Address1)); } return; }, TaskCreationOptions.LongRunning)); } Task.WaitAll(tasks.ToArray()); } }
public void PileSmallObjects(int payloadSize, params int[] freeChunkSizes) { using (var pile = new DefaultPile() { SegmentSize = PileCacheTestCore.SEG_SIZE, AllocMode = AllocationMode.ReuseSpace }) { pile.FreeChunkSizes = freeChunkSizes; pile.Start(); var pps = new List <PilePointer>(); while (pile.SegmentCount < 2) { pps.Add(pile.Put(generatePayload(payloadSize))); } pile.Delete(pps.Last()); pps.RemoveAt(pps.Count - 1); Console.WriteLine("just removed the last added payload and segment should be 1 now, real segment count is {0}", pile.SegmentCount); var objectsInFirstSegment = pps.Count; Console.WriteLine("put {0:N0} objects in first segment, pile.ObjectCount {1:N0}", objectsInFirstSegment, pile.ObjectCount); var deletedObjectCount = 0; for (int i = 0; i < pps.Count; i += 2, deletedObjectCount++) { Assert.IsTrue(pile.Delete(pps[i])); } Console.WriteLine("deleted {0:N0} objects, pile.ObjectCount {1:N0}", deletedObjectCount, pile.ObjectCount); var crawlStatus = pile.Crawl(false); Console.WriteLine("crawl: {0}", crawlStatus); var pps1 = new List <PilePointer>(); var c = 0; while (pile.SegmentCount < 3) { pps1.Add(pile.Put(generatePayload(payloadSize))); if (c % 20000 == 0) { pile.Crawl(true); //we do crawl because otherwise the 25000 free index xlots get exhausted AND } c++; //this unit tests does not run long enough to cause Crawl within allocator (5+ seconds) } //so we induce Crawl by hand to rebiild indexes pile.Delete(pps1.Last()); pps1.RemoveAt(pps1.Count - 1); Console.WriteLine("again just removed the last added payload and segment should be 2 now, real segment count is {0}", pile.SegmentCount); var objectsInSecondSegment = pps1.Count; Console.WriteLine("put {0:N0} objects in second segment, pile.ObjectCount {1:N0}", objectsInSecondSegment, pile.ObjectCount); Assert.AreEqual(objectsInFirstSegment * 2, pile.ObjectCount, "Object count in pile with two full segments must be equal to double object count in full first segment!"); } }
public static void PutGetDelete_Parallel(int fromSize, int toSize, int fromObjCount, int toObjCount, int taskCount) { Console.WriteLine("test will take about 1 minute"); int objectsPut = 0, objectsDeleted = 0;//, objectGot = 0; using (var pile = new DefaultPile(NOPApplication.Instance) { SegmentSize = SEG_SIZE }) { pile.Start(); Parallel.For(0, taskCount, _ => { var startTime = DateTime.Now; var objects = new Dictionary <PilePointer, byte[]>(); var objectCount = 0; var objectsSumSize = 0; var dtStop = DateTime.Now.AddMinutes(.5); while (dtStop >= DateTime.Now) { // insert routine var insertCount = Ambient.Random.NextScaledRandomInteger(fromObjCount, toObjCount); for (int i = 0; i < insertCount; i++) { var payloadSize = Ambient.Random.NextScaledRandomInteger(fromSize, toSize); var payload = new byte[payloadSize]; payload[0] = (byte)Ambient.Random.NextScaledRandomInteger(0, 255); payload[payloadSize - 1] = (byte)Ambient.Random.NextScaledRandomInteger(0, 255); var pp = pile.Put(payload); objects.Add(pp, payload); objectCount++; objectsSumSize += payloadSize; } Interlocked.Add(ref objectsPut, insertCount); // get if (objectCount > 0) { var getCount = Ambient.Random.NextScaledRandomInteger(5 * fromObjCount, 5 * toObjCount); for (int i = 0; i < getCount; i++) { var objectIdx = Ambient.Random.NextScaledRandomInteger(0, objectCount - 1); var obj = objects.ElementAt(objectIdx); var objPayloadFromPile = (byte[])pile.Get(obj.Key); Aver.AreEqual(obj.Value[0], objPayloadFromPile[0]); Aver.AreEqual(obj.Value[obj.Value.Length - 1], objPayloadFromPile[obj.Value.Length - 1]); Aver.AreEqual(obj.Value.Length, objPayloadFromPile.Length); } } // delete var deleteCount = Ambient.Random.NextScaledRandomInteger(fromObjCount, toObjCount); if (deleteCount > objectCount) { deleteCount = objectCount; } for (int i = 0; i < deleteCount; i++) { if (objectCount == 0) { break; } var objectIdx = Ambient.Random.NextScaledRandomInteger(0, objectCount - 1); var obj = objects.ElementAt(objectIdx); Aver.IsTrue(pile.Delete(obj.Key)); objects.Remove(obj.Key); objectCount--; objectsSumSize -= obj.Value.Length; } Interlocked.Add(ref objectsDeleted, deleteCount); } }); } Console.WriteLine("put {0:N0}, deleted {1:N0} object", objectsPut, objectsDeleted); }
public static void Chessboard_ByteArray(bool speed, int durationSec, int payloadSizeMin, int payloadSizeMax, bool isParallel) { using (var pile = new DefaultPile()) { pile.AllocMode = speed ? AllocationMode.FavorSpeed : AllocationMode.ReuseSpace; pile.Start(); var startTime = DateTime.UtcNow; var tasks = new List<Task>(); for (var t = 0; t < (isParallel ? (System.Environment.ProcessorCount - 1) : 1); t++) tasks.Add(Task.Factory.StartNew(() => { var list = new List<CheckByteArray>(); var i = 0; var wlc = 0; while (true) { if ((DateTime.UtcNow - startTime).TotalSeconds >= durationSec) break; var payloadSize = NFX.ExternalRandomGenerator .Instance.NextScaledRandomInteger(payloadSizeMin, payloadSizeMax); var val = new byte[payloadSize]; val[0] = (byte)NFX.ExternalRandomGenerator.Instance.NextRandomInteger; val[payloadSize - 1] = (byte)NFX.ExternalRandomGenerator.Instance.NextRandomInteger; var ptr = pile.Put(val); var element = new CheckByteArray(ptr, payloadSize - 1, val[0], val[payloadSize - 1]); list.Add(element); // delete previous element if (list.Count > 1 && i % 2 == 0) { ptr = list[list.Count - 2].Ptr; pile.Delete(ptr); list.RemoveAt(list.Count - 2); } // get several random elements if (list.Count > 64 && NFX.ExternalRandomGenerator.Instance.NextScaledRandomInteger(0, 100) > 98) { var toRead = NFX.ExternalRandomGenerator.Instance.NextScaledRandomInteger(8, 64); wlc++; if (wlc % 125 == 0) Console.WriteLine("Thread {0} is reading {1} elements, total {2}, Pile objects {3}, Pile segments {4} Pile Bytes {5}" .Args(Thread.CurrentThread.ManagedThreadId, toRead, list.Count, pile.ObjectCount, pile.SegmentCount, pile.AllocatedMemoryBytes)); for (var k = 0; k < toRead; k++) { element = list[NFX.ExternalRandomGenerator.Instance.NextScaledRandomInteger(0, list.Count - 1)]; var buf = pile.Get(element.Ptr) as byte[]; Assert.AreEqual(element.FirstByte, buf[0]); Assert.AreEqual(element.LastByte, buf[element.IdxLast]); } } if (i == Int32.MaxValue) i = 0; else i++; if (list.Count == Int32.MaxValue) list = new List<CheckByteArray>(); } // total check Console.WriteLine("Thread {0} is doing final read of {1} elements, ObjectCount {2}" .Args(Thread.CurrentThread.ManagedThreadId, list.Count, pile.ObjectCount)); foreach (var element in list) { var buf = pile.Get(element.Ptr) as byte[]; Assert.AreEqual(element.FirstByte, buf[0]); Assert.AreEqual(element.LastByte, buf[element.IdxLast]); } return; }, TaskCreationOptions.LongRunning)); Task.WaitAll(tasks.ToArray()); } }
public void TestThreadSafety(int cnt, int tcount, int seconds) { using (var pile = new DefaultPile()) { // pile.SegmentSize = 64 * 1024 * 1024; pile.Start(); var data = new PilePointer[cnt]; for (var i = 0; i < cnt; i++) { data[i] = pile.Put(new byte[0], preallocateBlockSize: Memory.PTR_RAW_BYTE_SIZE + (137 * (i % 17))); } var lst = new List <Task>(); long statRead = 0; long statPut = 0; long statDelete = 0; var sw = Stopwatch.StartNew(); var sd = DateTime.UtcNow; var deleteLock = new object(); for (var i = 0; i < tcount; i++) { lst.Add(Task.Factory.StartNew(() => { while (true) { var now = DateTime.UtcNow; if ((now - sd).TotalSeconds > seconds) { break; } var it = ExternalRandomGenerator.Instance.NextScaledRandomInteger(10, 100); for (var j = 0; j < it; j++) { var pp = data[ExternalRandomGenerator.Instance.NextScaledRandomInteger(0, cnt - 1)]; Aver.IsTrue(pile.Get(pp) is byte[]); Interlocked.Increment(ref statRead); } it = ExternalRandomGenerator.Instance.NextScaledRandomInteger(10, 100); for (var j = 0; j < it; j++) { var pp = data[ExternalRandomGenerator.Instance.NextScaledRandomInteger(0, cnt - 1)]; Aver.IsTrue(pile.Put(pp, new byte[ExternalRandomGenerator.Instance.NextScaledRandomInteger(0, 3791)], link: true)); Interlocked.Increment(ref statPut); } if (ExternalRandomGenerator.Instance.NextScaledRandomInteger(0, 100) > 50 && Monitor.TryEnter(deleteLock)) { try { var newData = (PilePointer[])data.Clone(); it = ExternalRandomGenerator.Instance.NextScaledRandomInteger(0, 10 + (cnt / 2)); var toDelete = new List <PilePointer>(); for (var j = 0; j < it; j++) { var idx = ExternalRandomGenerator.Instance.NextScaledRandomInteger(0, cnt - 1); toDelete.Add(newData[idx]); newData[idx] = pile.Put(new byte[12], preallocateBlockSize: ExternalRandomGenerator.Instance.NextScaledRandomInteger(24, 1024)); } data = newData;//atomic; Thread.Sleep(1000); foreach (var pp in toDelete) { Aver.IsTrue(pile.Delete(pp)); Interlocked.Increment(ref statDelete); } } finally { Monitor.Exit(deleteLock); } } } }, TaskCreationOptions.LongRunning)); } Task.WaitAll(lst.ToArray()); var el = sw.ElapsedMilliseconds; Console.WriteLine("Read {0:n0} at {1:n0} ops/sec".Args(statRead, statRead / (el / 1000d))); Console.WriteLine("Put {0:n0} at {1:n0} ops/sec".Args(statPut, statPut / (el / 1000d))); Console.WriteLine("Deleted {0:n0} at {1:n0} ops/sec".Args(statDelete, statDelete / (el / 1000d))); for (var i = 0; i < data.Length; i++) { Aver.IsTrue(pile.Delete(data[i])); } Aver.AreEqual(0, pile.ObjectCount); Aver.AreEqual(0, pile.ObjectLinkCount); Aver.AreEqual(0, pile.AllocatedMemoryBytes); } }
public static void PutGetDelete_Sequential(int fromSize, int toSize, int fromObjCount, int toObjCount) { Console.WriteLine("test will take about 1 minute"); var startTime = DateTime.Now; var objects = new Dictionary <PilePointer, byte[]>(); var objectCount = 0; var objectsSumSize = 0; using (var pile = new DefaultPile() { SegmentSize = PileCacheTestCore.SEG_SIZE }) { pile.Start(); var dtStop = DateTime.Now.AddMinutes(1); while (dtStop >= DateTime.Now) { // insert routine var insertCount = NFX.ExternalRandomGenerator.Instance.NextScaledRandomInteger(fromObjCount, toObjCount); for (int i = 0; i < insertCount; i++) { var payloadSize = NFX.ExternalRandomGenerator.Instance.NextScaledRandomInteger(fromSize, toSize); var payload = new byte[payloadSize]; payload[0] = (byte)NFX.ExternalRandomGenerator.Instance.NextScaledRandomInteger(0, 255); payload[payloadSize - 1] = (byte)NFX.ExternalRandomGenerator.Instance.NextScaledRandomInteger(0, 255); var pp = pile.Put(payload); objects.Add(pp, payload); objectCount++; objectsSumSize += payloadSize; } // get if (objectCount > 0) { var getCount = ExternalRandomGenerator.Instance.NextScaledRandomInteger(5 * fromObjCount, 5 * toObjCount); for (int i = 0; i < getCount; i++) { var objectIdx = ExternalRandomGenerator.Instance.NextScaledRandomInteger(0, objectCount - 1); var obj = objects.ElementAt(objectIdx); var objPayloadFromPile = (byte[])pile.Get(obj.Key); Assert.AreEqual(obj.Value[0], objPayloadFromPile[0]); Assert.AreEqual(obj.Value[obj.Value.Length - 1], objPayloadFromPile[obj.Value.Length - 1]); Assert.AreEqual(obj.Value.Length, objPayloadFromPile.Length); } } // delete var deleteCount = ExternalRandomGenerator.Instance.NextScaledRandomInteger(fromObjCount, toObjCount); for (int i = 0; i < deleteCount; i++) { if (objectCount == 0) { break; } var objectIdx = ExternalRandomGenerator.Instance.NextScaledRandomInteger(0, objectCount - 1); var obj = objects.ElementAt(objectIdx); Assert.IsTrue(pile.Delete(obj.Key)); objects.Remove(obj.Key); objectCount--; objectsSumSize -= obj.Value.Length; } } } }
public void VarSizes_Checkboard(bool isParallel, int cnt, int minSz, int maxSz, bool speed) { using (var pile = new DefaultPile()) { pile.Start(); pile.AllocMode = speed ? AllocationMode.FavorSpeed : AllocationMode.ReuseSpace; var tasks = new List <Task>(); for (var t = 0; t < (isParallel? (System.Environment.ProcessorCount - 1) : 1); t++) { tasks.Add( Task.Run(() => { var dict = new Dictionary <PilePointer, dummy>(); var priorOdd = PilePointer.Invalid; for (var i = 0; i < cnt; i++) { var even = (i & 0x01) == 0; var data = new dummy { bin = new byte[12 + NFX.ExternalRandomGenerator.Instance.NextScaledRandomInteger(minSz, maxSz)] }; data.bin.WriteBEInt32(0, ExternalRandomGenerator.Instance.NextRandomInteger); data.bin.WriteBEInt32(data.bin.Length - 4, ExternalRandomGenerator.Instance.NextRandomInteger); var ptr = pile.Put(data); Assert.IsTrue(ptr.Valid); if (even) { dict.Add(ptr, data); } else { if (priorOdd.Valid) { Assert.IsTrue(pile.Delete(priorOdd)); } priorOdd = ptr; } if (i % 1000 == 0) { Console.WriteLine("Thread{0} did {1}; allocated {2} bytes, utilized {3} bytes by {4} objects {5} bytes/obj. ", Thread.CurrentThread.ManagedThreadId, i, pile.AllocatedMemoryBytes, pile.UtilizedBytes, pile.ObjectCount, pile.UtilizedBytes / pile.ObjectCount); } } Console.WriteLine("Thread {0} Population done, now checking the buffers... {1}", Thread.CurrentThread.ManagedThreadId, DateTime.Now); foreach (var entry in dict) { Assert.IsTrue(NFX.IOMiscUtils.MemBufferEquals(entry.Value.bin, (pile.Get(entry.Key) as dummy).bin)); } Console.WriteLine("Thread {0} DONE. {1}", Thread.CurrentThread.ManagedThreadId, DateTime.Now); }) );//add } Task.WaitAll(tasks.ToArray()); } }
public void VarSizes_Increasing_Random(bool isParallel, int cnt, int minSz, int maxSz, bool speed, bool rnd) { using (var pile = new DefaultPile()) { pile.Start(); pile.AllocMode = speed ? AllocationMode.FavorSpeed : AllocationMode.ReuseSpace; var sw = Stopwatch.StartNew(); var tasks = new List <Task>(); for (var t = 0; t < (isParallel? (System.Environment.ProcessorCount - 1) : 1); t++) { tasks.Add( Task.Run(() => { var dict = new Dictionary <PilePointer, dummy>(); var lst = new List <PilePointer>(); var priorOdd = PilePointer.Invalid; for (var i = 0; i < cnt; i++) { var buf = new byte[12 + minSz + ( rnd ? (NFX.ExternalRandomGenerator.Instance.NextScaledRandomInteger(0, (int)(maxSz * (i / (double)cnt)))) : (int)(maxSz * (i / (double)cnt)) ) ]; var data = new dummy { bin = buf }; data.bin.WriteBEInt32(0, ExternalRandomGenerator.Instance.NextRandomInteger); data.bin.WriteBEInt32(data.bin.Length - 4, ExternalRandomGenerator.Instance.NextRandomInteger); var ptr = pile.Put(data); Assert.IsTrue(ptr.Valid); dict.Add(ptr, data); lst.Add(ptr); if (i > cnt / 3) { if (ExternalRandomGenerator.Instance.NextRandomInteger > 0) { var ri = ExternalRandomGenerator.Instance.NextScaledRandomInteger(0, lst.Count - 1); var pp = lst[ri]; if (!pp.Valid) { continue; } Assert.IsTrue(pile.Delete(pp)); dict.Remove(pp); lst[ri] = PilePointer.Invalid; } } if (i % 1000 == 0) { Console.WriteLine("Thread{0} did {1}; allocated {2} bytes, utilized {3} bytes by {4} objects {5} bytes/obj. ", Thread.CurrentThread.ManagedThreadId, i, pile.AllocatedMemoryBytes, pile.UtilizedBytes, pile.ObjectCount, pile.UtilizedBytes / pile.ObjectCount); } } Console.WriteLine("Thread {0} Population done, now checking the buffers... {1}", Thread.CurrentThread.ManagedThreadId, DateTime.Now); foreach (var entry in dict) { Assert.IsTrue(NFX.IOMiscUtils.MemBufferEquals(entry.Value.bin, (pile.Get(entry.Key) as dummy).bin)); } Console.WriteLine("Thread {0} DONE. {1}", Thread.CurrentThread.ManagedThreadId, DateTime.Now); }) );//add } Task.WaitAll(tasks.ToArray()); var el = sw.ElapsedMilliseconds; var gt = cnt * tasks.Count; Console.WriteLine("Total objects: {0:n0} in {1:n0} ms at {2:n0} obj/sec".Args(gt, el, gt / (el / 1000d))); } }
public static void Chessboard_TRow(bool speed, int durationSec, bool isParallel) { using (var pile = new DefaultPile(NOPApplication.Instance)) { pile.AllocMode = speed ? AllocationMode.FavorSpeed : AllocationMode.ReuseSpace; pile.Start(); var startTime = DateTime.UtcNow; var tasks = new List <Task>(); for (var t = 0; t < (isParallel ? (System.Environment.ProcessorCount - 1) : 1); t++) { tasks.Add(Task.Factory.StartNew(() => { var list = new List <CheckTRow>(); var i = 0; var wlc = 0; while (true) { if ((DateTime.UtcNow - startTime).TotalSeconds >= durationSec) { break; } var val = PersonRow.MakeFake(new GDID(0, (ulong)i)); var ptr = pile.Put(val); var element = new CheckTRow(ptr, val.ID, val.Address1); list.Add(element); // delete previous element if (list.Count > 1 && i % 2 == 0) { ptr = list[list.Count - 2].Ptr; pile.Delete(ptr); list.RemoveAt(list.Count - 2); } // get several random elements if (list.Count > 64 && Ambient.Random.NextScaledRandomInteger(0, 100) > 98) { var toRead = Ambient.Random.NextScaledRandomInteger(8, 64); wlc++; if (wlc % 125 == 0) { Console.WriteLine("Thread {0} is reading {1} elements, total {2}, Pile objects {3}, Pile segments {4} Pile Bytes {5}" .Args(Thread.CurrentThread.ManagedThreadId, toRead, list.Count, pile.ObjectCount, pile.SegmentCount, pile.AllocatedMemoryBytes)); } for (var k = 0; k < toRead; k++) { element = list[Ambient.Random.NextScaledRandomInteger(0, list.Count - 1)]; var buf = pile.Get(element.Ptr) as PersonRow; Aver.IsTrue(element.Id.Equals(buf.ID)); Aver.IsTrue(element.Address.Equals(buf.Address1)); } } if (i == Int32.MaxValue) { i = 0; } else { i++; } if (list.Count == Int32.MaxValue) { list = new List <CheckTRow>(); } } // total check Console.WriteLine("Thread {0} is doing final read of {1} elements, objectCount {2}" .Args(Thread.CurrentThread.ManagedThreadId, list.Count, pile.ObjectCount)); foreach (var element in list) { var buf = pile.Get(element.Ptr) as PersonRow; Aver.IsTrue(element.Id.Equals(buf.ID)); Aver.IsTrue(element.Address.Equals(buf.Address1)); } return; }, TaskCreationOptions.LongRunning)); } Task.WaitAll(tasks.ToArray()); } }
public static void Put_RandomDelete_ByteArray(int cnt, int durationSec, bool speed, int payloadSizeMin, int payloadSizeMax, int deleteFreq, bool isParallel) { using (var pile = new DefaultPile()) { pile.AllocMode = speed ? AllocationMode.FavorSpeed : AllocationMode.ReuseSpace; pile.Start(); var startTime = DateTime.UtcNow; var tasks = new List <Task>(); for (var t = 0; t < (isParallel ? (System.Environment.ProcessorCount - 1) : 1); t++) { tasks.Add(Task.Factory.StartNew(() => { var wlc = 0; while (true) { if ((DateTime.UtcNow - startTime).TotalSeconds >= durationSec) { return; } var dict = new Dictionary <int, CheckByteArray>(); Console.WriteLine("Starting a batch of {0}".Args(cnt)); for (int i = 0; i < cnt; i++) { var payloadSize = NFX.ExternalRandomGenerator.Instance.NextScaledRandomInteger(payloadSizeMin, payloadSizeMax); var val = new byte[payloadSize]; val[0] = (byte)NFX.ExternalRandomGenerator.Instance.NextRandomInteger; val[payloadSize - 1] = (byte)NFX.ExternalRandomGenerator.Instance.NextRandomInteger; var ptr = pile.Put(val); var element = new CheckByteArray(ptr, payloadSize - 1, val[0], val[payloadSize - 1]); dict.Add(i, element); if (dict.Count > 0 && i % deleteFreq == 0) { while (true) { var idx = i - NFX.ExternalRandomGenerator.Instance.NextScaledRandomInteger(0, i); CheckByteArray stored; if (dict.TryGetValue(idx, out stored)) { ptr = stored.Ptr; pile.Delete(ptr); dict.Remove(idx); break; } } } if (dict.Count > 16 && NFX.ExternalRandomGenerator.Instance.NextScaledRandomInteger(0, 100) > 98) { var toRead = NFX.ExternalRandomGenerator.Instance.NextScaledRandomInteger(8, 64); wlc++; if (wlc % 125 == 0) { Console.WriteLine("Thread {0} is reading {1} elements, total {2}" .Args(Thread.CurrentThread.ManagedThreadId, toRead, dict.Count)); } for (var k = 0; k < toRead; k++) { var kvp = dict.Skip(NFX.ExternalRandomGenerator.Instance.NextScaledRandomInteger(0, dict.Count - 1)).First(); var buf = pile.Get(kvp.Value.Ptr) as byte[]; Aver.AreEqual(kvp.Value.FirstByte, buf[0]); Aver.AreEqual(kvp.Value.LastByte, buf[kvp.Value.IdxLast]); } } } Console.WriteLine("Thread {0} is doing final read of {1} elements".Args(Thread.CurrentThread.ManagedThreadId, dict.Count)); foreach (var kvp in dict) { var buf = pile.Get(kvp.Value.Ptr) as byte[]; Aver.AreEqual(kvp.Value.FirstByte, buf[0]); Aver.AreEqual(kvp.Value.LastByte, buf[kvp.Value.IdxLast]); } } }, TaskCreationOptions.LongRunning)); } Task.WaitAll(tasks.ToArray()); } }
public static void DeleteSeveral_ByteArray(bool speed, int durationSec, int putMin, int putMax, int delFactor, int payloadSizeMin, int payloadSizeMax, bool isParallel) { using (var pile = new DefaultPile()) { pile.AllocMode = speed ? AllocationMode.FavorSpeed : AllocationMode.ReuseSpace; pile.Start(); var startTime = DateTime.UtcNow; var tasks = new List <Task>(); for (var t = 0; t < (isParallel ? (System.Environment.ProcessorCount - 1) : 1); t++) { tasks.Add(Task.Factory.StartNew(() => { var list = new List <CheckByteArray>(); var wlc = 0; while (true) { if ((DateTime.UtcNow - startTime).TotalSeconds >= durationSec) { break; } var putCount = NFX.ExternalRandomGenerator.Instance.NextScaledRandomInteger(putMin, putMax); for (int i = 0; i < putCount; i++) { var payloadSize = NFX.ExternalRandomGenerator .Instance.NextScaledRandomInteger(payloadSizeMin, payloadSizeMax); var val = new byte[payloadSize]; val[0] = (byte)NFX.ExternalRandomGenerator.Instance.NextRandomInteger; val[payloadSize - 1] = (byte)NFX.ExternalRandomGenerator.Instance.NextRandomInteger; var ptr = pile.Put(val); list.Add(new CheckByteArray(ptr, payloadSize - 1, val[0], val[payloadSize - 1])); } int delCount = putCount / delFactor; for (int i = 0; i < delCount; i++) { var idx = NFX.ExternalRandomGenerator.Instance.NextScaledRandomInteger(0, list.Count - 1); var ptr = list[idx].Ptr; pile.Delete(ptr); list.RemoveAt(idx); } // get several random elements if (list.Count > 64 && NFX.ExternalRandomGenerator.Instance.NextScaledRandomInteger(0, 100) > 98) { var toRead = NFX.ExternalRandomGenerator.Instance.NextScaledRandomInteger(8, 64); wlc++; if (wlc % 125 == 0) { Console.WriteLine("Thread {0} is reading {1} elements, total {2}" .Args(Thread.CurrentThread.ManagedThreadId, toRead, list.Count)); } for (var k = 0; k < toRead; k++) { var element = list[NFX.ExternalRandomGenerator.Instance.NextScaledRandomInteger(0, list.Count - 1)]; var buf = pile.Get(element.Ptr) as byte[]; Aver.AreEqual(element.FirstByte, buf[0]); Aver.AreEqual(element.LastByte, buf[element.IdxLast]); } } if (list.Count == Int32.MaxValue) { list = new List <CheckByteArray>(); } } // total check Console.WriteLine("Thread {0} is doing final read of {1} elements, objectCount {2}" .Args(Thread.CurrentThread.ManagedThreadId, list.Count, pile.ObjectCount)); foreach (var element in list) { var buf = pile.Get(element.Ptr) as byte[]; Aver.AreEqual(element.FirstByte, buf[0]); Aver.AreEqual(element.LastByte, buf[element.IdxLast]); } return; })); } Task.WaitAll(tasks.ToArray()); } }