public void VarSizes_Checkboard(bool isParallel, int cnt, int minSz, int maxSz, bool speed) { using (var pile = new DefaultPile()) { pile.Start(); pile.AllocMode = speed ? AllocationMode.FavorSpeed : AllocationMode.ReuseSpace; var tasks = new List <Task>(); for (var t = 0; t < (isParallel? (System.Environment.ProcessorCount - 1) : 1); t++) { tasks.Add( Task.Run(() => { var dict = new Dictionary <PilePointer, dummy>(); var priorOdd = PilePointer.Invalid; for (var i = 0; i < cnt; i++) { var even = (i & 0x01) == 0; var data = new dummy { bin = new byte[12 + NFX.ExternalRandomGenerator.Instance.NextScaledRandomInteger(minSz, maxSz)] }; data.bin.WriteBEInt32(0, ExternalRandomGenerator.Instance.NextRandomInteger); data.bin.WriteBEInt32(data.bin.Length - 4, ExternalRandomGenerator.Instance.NextRandomInteger); var ptr = pile.Put(data); Assert.IsTrue(ptr.Valid); if (even) { dict.Add(ptr, data); } else { if (priorOdd.Valid) { Assert.IsTrue(pile.Delete(priorOdd)); } priorOdd = ptr; } if (i % 1000 == 0) { Console.WriteLine("Thread{0} did {1}; allocated {2} bytes, utilized {3} bytes by {4} objects {5} bytes/obj. ", Thread.CurrentThread.ManagedThreadId, i, pile.AllocatedMemoryBytes, pile.UtilizedBytes, pile.ObjectCount, pile.UtilizedBytes / pile.ObjectCount); } } Console.WriteLine("Thread {0} Population done, now checking the buffers... {1}", Thread.CurrentThread.ManagedThreadId, DateTime.Now); foreach (var entry in dict) { Assert.IsTrue(NFX.IOMiscUtils.MemBufferEquals(entry.Value.bin, (pile.Get(entry.Key) as dummy).bin)); } Console.WriteLine("Thread {0} DONE. {1}", Thread.CurrentThread.ManagedThreadId, DateTime.Now); }) );//add } Task.WaitAll(tasks.ToArray()); } }
public static void Chessboard_TRow(bool speed, int durationSec, bool isParallel) { using (var pile = new DefaultPile()) { pile.AllocMode = speed ? AllocationMode.FavorSpeed : AllocationMode.ReuseSpace; pile.Start(); var startTime = DateTime.UtcNow; var tasks = new List <Task>(); for (var t = 0; t < (isParallel ? (System.Environment.ProcessorCount - 1) : 1); t++) { tasks.Add(Task.Factory.StartNew(() => { var list = new List <CheckTRow>(); var i = 0; var wlc = 0; while (true) { if ((DateTime.UtcNow - startTime).TotalSeconds >= durationSec) { break; } var val = PersonRow.MakeFake(new GDID(0, (ulong)i)); var ptr = pile.Put(val); var element = new CheckTRow(ptr, val.ID, val.Address1); list.Add(element); // delete previous element if (list.Count > 1 && i % 2 == 0) { ptr = list[list.Count - 2].Ptr; pile.Delete(ptr); list.RemoveAt(list.Count - 2); } // get several random elements if (list.Count > 64 && NFX.ExternalRandomGenerator.Instance.NextScaledRandomInteger(0, 100) > 98) { var toRead = NFX.ExternalRandomGenerator.Instance.NextScaledRandomInteger(8, 64); wlc++; if (wlc % 125 == 0) { Console.WriteLine("Thread {0} is reading {1} elements, total {2}, Pile objects {3}, Pile segments {4} Pile Bytes {5}" .Args(Thread.CurrentThread.ManagedThreadId, toRead, list.Count, pile.ObjectCount, pile.SegmentCount, pile.AllocatedMemoryBytes)); } for (var k = 0; k < toRead; k++) { element = list[NFX.ExternalRandomGenerator.Instance.NextScaledRandomInteger(0, list.Count - 1)]; var buf = pile.Get(element.Ptr) as PersonRow; Assert.IsTrue(element.Id.Equals(buf.ID)); Assert.IsTrue(element.Address.Equals(buf.Address1)); } } if (i == Int32.MaxValue) { i = 0; } else { i++; } if (list.Count == Int32.MaxValue) { list = new List <CheckTRow>(); } } // total check Console.WriteLine("Thread {0} is doing final read of {1} elements, objectCount {2}" .Args(Thread.CurrentThread.ManagedThreadId, list.Count, pile.ObjectCount)); foreach (var element in list) { var buf = pile.Get(element.Ptr) as PersonRow; Assert.IsTrue(element.Id.Equals(buf.ID)); Assert.IsTrue(element.Address.Equals(buf.Address1)); } return; }, TaskCreationOptions.LongRunning)); } Task.WaitAll(tasks.ToArray()); } }
public static void NoGrowth_ByteArray(bool speed, int durationSec, int payloadSizeMin, int payloadSizeMax, int countMin, int countMax) { using (var pile = new DefaultPile()) { pile.AllocMode = speed ? AllocationMode.FavorSpeed : AllocationMode.ReuseSpace; pile.Start(); var startTime = DateTime.UtcNow; var tasks = new List <Task>(); for (var t = 0; t < (System.Environment.ProcessorCount - 1); t++) { tasks.Add(Task.Factory.StartNew(() => { var list = new List <CheckByteArray>(); bool put = true; while (true) { if ((DateTime.UtcNow - startTime).TotalSeconds >= durationSec) { return; } if (put) { var cnt = NFX.ExternalRandomGenerator.Instance.NextScaledRandomInteger(countMin, countMax); for (int j = 0; j < cnt; j++) { var payloadSize = NFX.ExternalRandomGenerator .Instance.NextScaledRandomInteger(payloadSizeMin, payloadSizeMax); var val = new byte[payloadSize]; val[0] = (byte)NFX.ExternalRandomGenerator.Instance.NextRandomInteger; val[payloadSize - 1] = (byte)NFX.ExternalRandomGenerator.Instance.NextRandomInteger; var ptr = pile.Put(val); var element = new CheckByteArray(ptr, payloadSize - 1, val[0], val[payloadSize - 1]); list.Add(element); } Console.WriteLine("Thread {0} put {1} objects".Args(Thread.CurrentThread.ManagedThreadId, list.Count)); put = false; } else { Console.WriteLine("Thread {0} deleted {1} objects".Args(Thread.CurrentThread.ManagedThreadId, list.Count)); for (var j = 0; j < list.Count; j++) { var element = list[j]; var buf = pile.Get(element.Ptr) as byte[]; Assert.AreEqual(element.FirstByte, buf[0]); Assert.AreEqual(element.LastByte, buf[element.IdxLast]); pile.Delete(element.Ptr); } list.Clear(); put = true; } } }, TaskCreationOptions.LongRunning)); } Task.WaitAll(tasks.ToArray()); } Console.WriteLine("Test finished."); }
public static void Chessboard_ByteArray(bool speed, int durationSec, int payloadSizeMin, int payloadSizeMax, bool isParallel) { using (var pile = new DefaultPile()) { pile.AllocMode = speed ? AllocationMode.FavorSpeed : AllocationMode.ReuseSpace; pile.Start(); var startTime = DateTime.UtcNow; var tasks = new List<Task>(); for (var t = 0; t < (isParallel ? (System.Environment.ProcessorCount - 1) : 1); t++) tasks.Add(Task.Factory.StartNew(() => { var list = new List<CheckByteArray>(); var i = 0; var wlc = 0; while (true) { if ((DateTime.UtcNow - startTime).TotalSeconds >= durationSec) break; var payloadSize = NFX.ExternalRandomGenerator .Instance.NextScaledRandomInteger(payloadSizeMin, payloadSizeMax); var val = new byte[payloadSize]; val[0] = (byte)NFX.ExternalRandomGenerator.Instance.NextRandomInteger; val[payloadSize - 1] = (byte)NFX.ExternalRandomGenerator.Instance.NextRandomInteger; var ptr = pile.Put(val); var element = new CheckByteArray(ptr, payloadSize - 1, val[0], val[payloadSize - 1]); list.Add(element); // delete previous element if (list.Count > 1 && i % 2 == 0) { ptr = list[list.Count - 2].Ptr; pile.Delete(ptr); list.RemoveAt(list.Count - 2); } // get several random elements if (list.Count > 64 && NFX.ExternalRandomGenerator.Instance.NextScaledRandomInteger(0, 100) > 98) { var toRead = NFX.ExternalRandomGenerator.Instance.NextScaledRandomInteger(8, 64); wlc++; if (wlc % 125 == 0) Console.WriteLine("Thread {0} is reading {1} elements, total {2}, Pile objects {3}, Pile segments {4} Pile Bytes {5}" .Args(Thread.CurrentThread.ManagedThreadId, toRead, list.Count, pile.ObjectCount, pile.SegmentCount, pile.AllocatedMemoryBytes)); for (var k = 0; k < toRead; k++) { element = list[NFX.ExternalRandomGenerator.Instance.NextScaledRandomInteger(0, list.Count - 1)]; var buf = pile.Get(element.Ptr) as byte[]; Assert.AreEqual(element.FirstByte, buf[0]); Assert.AreEqual(element.LastByte, buf[element.IdxLast]); } } if (i == Int32.MaxValue) i = 0; else i++; if (list.Count == Int32.MaxValue) list = new List<CheckByteArray>(); } // total check Console.WriteLine("Thread {0} is doing final read of {1} elements, ObjectCount {2}" .Args(Thread.CurrentThread.ManagedThreadId, list.Count, pile.ObjectCount)); foreach (var element in list) { var buf = pile.Get(element.Ptr) as byte[]; Assert.AreEqual(element.FirstByte, buf[0]); Assert.AreEqual(element.LastByte, buf[element.IdxLast]); } return; }, TaskCreationOptions.LongRunning)); Task.WaitAll(tasks.ToArray()); } }
public void PileSmallObjects(int payloadSize, params int[] freeChunkSizes) { using (var pile = new DefaultPile() { SegmentSize = PileCacheTestCore.SEG_SIZE, AllocMode = AllocationMode.ReuseSpace }) { pile.FreeChunkSizes = freeChunkSizes; pile.Start(); var pps = new List <PilePointer>(); while (pile.SegmentCount < 2) { pps.Add(pile.Put(generatePayload(payloadSize))); } pile.Delete(pps.Last()); pps.RemoveAt(pps.Count - 1); Console.WriteLine("Just removed the last added payload and segment should be 1 now, real segment count is {0}", pile.SegmentCount); Aver.AreEqual(1, pile.SegmentCount); var objectsInFirstSegment = pps.Count; Console.WriteLine("put {0:N0} objects in first segment, pile.ObjectCount {1:N0}", objectsInFirstSegment, pile.ObjectCount); var deletedObjectCount = 0; for (int i = 0; i < pps.Count; i += 2, deletedObjectCount++) { Aver.IsTrue(pile.Delete(pps[i])); } var objectsInFirstSegmentAfterDelete = pile.ObjectCount; Console.WriteLine("Deleted {0:N0} objects from 1st segment, pile.ObjectCount {1:N0}", deletedObjectCount, pile.ObjectCount); Aver.AreEqual(objectsInFirstSegment / 2, objectsInFirstSegmentAfterDelete); Console.WriteLine("---------------------------------------------------------"); var crawlStatus = pile.Crawl(false); Console.WriteLine("crawl: {0}", crawlStatus); var pps1 = new List <PilePointer>(); var c = 0; while (pile.SegmentCount < 3) { pps1.Add(pile.Put(generatePayload(payloadSize))); if (c % 20000 == 0) { pile.Crawl(true); //we do crawl because otherwise the 25000 free index xlots get exhausted AND } c++; //this unit tests does not run long enough to cause Crawl within allocator (5+ seconds) } //so we induce Crawl by hand to rebiild indexes pile.Delete(pps1.Last()); pps1.RemoveAt(pps1.Count - 1); Console.WriteLine("Again just removed the last added payload and segment should be 2 now, real segment count is {0}", pile.SegmentCount); Aver.AreEqual(2, pile.SegmentCount); var objectsInSecondRound = pps1.Count; Console.WriteLine("Put {0:N0} objects in second round, pile.ObjectCount {1:N0}", objectsInSecondRound, pile.ObjectCount); Aver.AreWithin(objectsInFirstSegment + objectsInFirstSegmentAfterDelete, objectsInSecondRound, 2d, "#1"); Aver.AreEqual(objectsInFirstSegmentAfterDelete + objectsInSecondRound, pile.ObjectCount, "#2"); } }
public static void Put_RandomDelete_ByteArray(int cnt, int durationSec, bool speed, int payloadSizeMin, int payloadSizeMax, int deleteFreq, bool isParallel) { using (var pile = new DefaultPile(NOPApplication.Instance)) { pile.AllocMode = speed ? AllocationMode.FavorSpeed : AllocationMode.ReuseSpace; pile.Start(); var startTime = DateTime.UtcNow; var tasks = new List <Task>(); for (var t = 0; t < (isParallel ? (System.Environment.ProcessorCount - 1) : 1); t++) { tasks.Add(Task.Factory.StartNew(() => { var wlc = 0; while (true) { if ((DateTime.UtcNow - startTime).TotalSeconds >= durationSec) { return; } var dict = new Dictionary <int, CheckByteArray>(); Console.WriteLine("Starting a batch of {0}".Args(cnt)); for (int i = 0; i < cnt; i++) { var payloadSize = Ambient.Random.NextScaledRandomInteger(payloadSizeMin, payloadSizeMax); var val = new byte[payloadSize]; val[0] = (byte)Ambient.Random.NextRandomInteger; val[payloadSize - 1] = (byte)Ambient.Random.NextRandomInteger; var ptr = pile.Put(val); var element = new CheckByteArray(ptr, payloadSize - 1, val[0], val[payloadSize - 1]); dict.Add(i, element); if (dict.Count > 0 && i % deleteFreq == 0) { while (true) { var idx = i - Ambient.Random.NextScaledRandomInteger(0, i); CheckByteArray stored; if (dict.TryGetValue(idx, out stored)) { ptr = stored.Ptr; pile.Delete(ptr); dict.Remove(idx); break; } } } if (dict.Count > 16 && Ambient.Random.NextScaledRandomInteger(0, 100) > 98) { var toRead = Ambient.Random.NextScaledRandomInteger(8, 64); wlc++; if (wlc % 125 == 0) { Console.WriteLine("Thread {0} is reading {1} elements, total {2}" .Args(Thread.CurrentThread.ManagedThreadId, toRead, dict.Count)); } for (var k = 0; k < toRead; k++) { var kvp = dict.Skip(Ambient.Random.NextScaledRandomInteger(0, dict.Count - 1)).First(); var buf = pile.Get(kvp.Value.Ptr) as byte[]; Aver.AreEqual(kvp.Value.FirstByte, buf[0]); Aver.AreEqual(kvp.Value.LastByte, buf[kvp.Value.IdxLast]); } } } Console.WriteLine("Thread {0} is doing final read of {1} elements".Args(Thread.CurrentThread.ManagedThreadId, dict.Count)); foreach (var kvp in dict) { var buf = pile.Get(kvp.Value.Ptr) as byte[]; Aver.AreEqual(kvp.Value.FirstByte, buf[0]); Aver.AreEqual(kvp.Value.LastByte, buf[kvp.Value.IdxLast]); } } }, TaskCreationOptions.LongRunning)); } Task.WaitAll(tasks.ToArray()); } }
public static void DeleteSeveral_TRow(bool speed, int durationSec, int putMin, int putMax, int delFactor, bool isParallel) { using (var pile = new DefaultPile(NOPApplication.Instance)) { pile.AllocMode = speed ? AllocationMode.FavorSpeed : AllocationMode.ReuseSpace; pile.Start(); var startTime = DateTime.UtcNow; var tasks = new List <Task>(); for (var t = 0; t < (isParallel ? (System.Environment.ProcessorCount - 1) : 1); t++) { tasks.Add(Task.Factory.StartNew(() => { var list = new List <CheckTRow>(); var wlc = 0; while (true) { if ((DateTime.UtcNow - startTime).TotalSeconds >= durationSec) { break; } var putCount = Ambient.Random.NextScaledRandomInteger(putMin, putMax); for (int i = 0; i < putCount; i++) { var val = PersonRow.MakeFake(new GDID()); var ptr = pile.Put(val); list.Add(new CheckTRow(ptr, val.ID, val.Address1)); } // delete several random elements int delCount = putCount / delFactor; for (int i = 0; i < delCount; i++) { var idx = Ambient.Random.NextScaledRandomInteger(0, list.Count - 1); var ptr = list[idx].Ptr; pile.Delete(ptr); list.RemoveAt(idx); } // get several random elements if (list.Count > 64 && Ambient.Random.NextScaledRandomInteger(0, 100) > 98) { var toRead = Ambient.Random.NextScaledRandomInteger(8, 64); wlc++; if (wlc % 125 == 0) { Console.WriteLine("Thread {0} is reading {1} elements, total {2}" .Args(Thread.CurrentThread.ManagedThreadId, toRead, list.Count)); } for (var k = 0; k < toRead; k++) { var element = list[Ambient.Random.NextScaledRandomInteger(0, list.Count - 1)]; var buf = pile.Get(element.Ptr) as PersonRow; Aver.IsTrue(element.Id.Equals(buf.ID)); Aver.IsTrue(element.Address.Equals(buf.Address1)); } } } // total check Console.WriteLine("Thread {0} is doing final read of {1} elements, objectCount {2}" .Args(Thread.CurrentThread.ManagedThreadId, list.Count, pile.ObjectCount)); foreach (var element in list) { var buf = pile.Get(element.Ptr) as PersonRow; Aver.IsTrue(element.Id.Equals(buf.ID)); Aver.IsTrue(element.Address.Equals(buf.Address1)); } return; }, TaskCreationOptions.LongRunning)); } Task.WaitAll(tasks.ToArray()); } }
public void NoGrowth_ByteArray(bool speed, int durationSec, int payloadSizeMin, int payloadSizeMax, int countMin, int countMax) { using (var cache = new LocalCache(NOPApplication.Instance)) using (var pile = new DefaultPile(cache)) { cache.Pile = pile; cache.PileAllocMode = speed ? AllocationMode.FavorSpeed : AllocationMode.ReuseSpace; cache.Start(); var startTime = DateTime.UtcNow; var tasks = new List <Task>(); for (var t = 0; t < (System.Environment.ProcessorCount - 1); t++) { tasks.Add(Task.Factory.StartNew(() => { var tA = cache.GetOrCreateTable <GDID>("A"); var list = new List <CheckByteArray>(); bool put = true; while (true) { if ((DateTime.UtcNow - startTime).TotalSeconds >= durationSec) { return; } if (put) { var cnt = Ambient.Random.NextScaledRandomInteger(countMin, countMax); for (int i = 0; i < cnt; i++) { var payloadSize = Ambient.Random.NextScaledRandomInteger(payloadSizeMin, payloadSizeMax); var val = new byte[payloadSize]; val[0] = (byte)Ambient.Random.NextRandomInteger; val[payloadSize - 1] = (byte)Ambient.Random.NextRandomInteger; var key = new GDID((uint)Thread.CurrentThread.ManagedThreadId, (ulong)i); tA.Put(key, val); var element = new CheckByteArray(key, payloadSize - 1, val[0], val[payloadSize - 1]); list.Add(element); } Console.WriteLine("Thread {0} put {1} objects".Args(Thread.CurrentThread.ManagedThreadId, list.Count)); put = false; } else { var i = 0; for (var j = 0; j < list.Count; j++) { var element = list[j]; var buf = tA.Get(element.Key) as byte[]; if (buf != null) { Aver.AreEqual(element.FirstByte, buf[0]); Aver.AreEqual(element.LastByte, buf[element.IdxLast]); tA.Remove(element.Key); i++; } } Console.WriteLine("Thread {0} deleted {1} objects".Args(Thread.CurrentThread.ManagedThreadId, i)); list.Clear(); put = true; } } }, TaskCreationOptions.LongRunning)); } Task.WaitAll(tasks.ToArray()); } }
public void DeleteOne_ByteArray(bool speed, int durationSec, int payloadSizeMin, int payloadSizeMax, int deleteFreq, bool isParallel) { using (var cache = new LocalCache(NOPApplication.Instance)) using (var pile = new DefaultPile(cache)) { cache.Pile = pile; cache.PileAllocMode = speed ? AllocationMode.FavorSpeed : AllocationMode.ReuseSpace; cache.Start(); var startTime = DateTime.UtcNow; var tasks = new List <Task>(); for (var t = 0; t < (isParallel ? (System.Environment.ProcessorCount - 1) : 1); t++) { tasks.Add(Task.Factory.StartNew(() => { var i = 0; var list = new List <CheckByteArray>(); var tA = cache.GetOrCreateTable <GDID>("A"); var wlc = 0; while (true) { if ((DateTime.UtcNow - startTime).TotalSeconds >= durationSec) { break; } var payloadSize = Ambient.Random.NextScaledRandomInteger(payloadSizeMin, payloadSizeMax); var val = new byte[payloadSize]; val[0] = (byte)Ambient.Random.NextRandomInteger; val[payloadSize - 1] = (byte)Ambient.Random.NextRandomInteger; var key = new GDID((uint)Thread.CurrentThread.ManagedThreadId, (ulong)i); tA.Put(key, val); list.Add(new CheckByteArray(key, payloadSize - 1, val[0], val[payloadSize - 1])); // delete ONE random element if (i > 0 && i % deleteFreq == 0) { while (true && list.Count > 0) { var idx = Ambient.Random.NextScaledRandomInteger(0, list.Count - 1); key = list[idx].Key; var removed = tA.Remove(key); list.RemoveAt(idx); if (removed) { break; } } } // get several random elements if (list.Count > 64 && Ambient.Random.NextScaledRandomInteger(0, 100) > 98) { var toRead = Ambient.Random.NextScaledRandomInteger(8, 64); wlc++; if (wlc % 125 == 0) { Console.WriteLine("Thread {0} is reading {1} elements, total {2}" .Args(Thread.CurrentThread.ManagedThreadId, toRead, list.Count)); } for (var k = 0; k < toRead && list.Count > 0; k++) { var idx = Ambient.Random.NextScaledRandomInteger(0, list.Count - 1); var element = list[idx]; var buf = tA.Get(element.Key) as byte[]; if (buf == null) { list.RemoveAt(idx); continue; } Aver.AreEqual(element.FirstByte, buf[0]); Aver.AreEqual(element.LastByte, buf[element.IdxLast]); } } if (i == Int32.MaxValue) { i = 0; } else { i++; } if (list.Count == Int32.MaxValue) { list = new List <CheckByteArray>(); } } Console.WriteLine("Thread {0} is doing final read of {1} elements, tableCount {2}" .Args(Thread.CurrentThread.ManagedThreadId, list.Count, tA.Count)); foreach (var element in list) { var buf = tA.Get(element.Key) as byte[]; if (buf == null) { continue; } Aver.AreEqual(element.FirstByte, buf[0]); Aver.AreEqual(element.LastByte, buf[element.IdxLast]); } }, TaskCreationOptions.LongRunning)); } Task.WaitAll(tasks.ToArray()); } }
public void Parallel_PutGetDelete_Random() { const int PUTTER_CNT = 2, PUTTER_OP_CNT = 2 * 10000; const int GETTER_CNT = 6, GETTER_OP_CNT = 2 * 30000; const int DELETER_CNT = 2, DELETER_OP_CNT = 2 * 10000; var data = new ConcurrentDictionary <PilePointer, string>(); var getAccessViolations = new ConcurrentDictionary <int, int>(); var deleteAccessViolations = new ConcurrentDictionary <int, int>(); using (var pile = new DefaultPile()) { pile.Start(); var ipile = pile as IPile; // putter tasks var putters = new Task[PUTTER_CNT]; for (int it = 0; it < PUTTER_CNT; it++) { var task = new Task(() => { for (int i = 0; i < PUTTER_OP_CNT; i++) { var str = NFX.Parsing.NaturalTextGenerator.Generate(); var pp = ipile.Put(str); data.TryAdd(pp, str); } }); putters[it] = task; } // getter tasks var getters = new Task[GETTER_CNT]; for (int it = 0; it < GETTER_CNT; it++) { var task = new Task(() => { for (int i = 0; i < GETTER_OP_CNT; i++) { if (data.Count == 0) { System.Threading.Thread.Yield(); continue; } var idx = ExternalRandomGenerator.Instance.NextScaledRandomInteger(0, data.Count - 1); var kvp = data.ElementAt(idx); try { var str = ipile.Get(kvp.Key); Assert.AreEqual(str, kvp.Value); } catch (PileAccessViolationException) { getAccessViolations.AddOrUpdate(System.Threading.Thread.CurrentThread.ManagedThreadId, 1, (mid, val) => val + 1); } } }); getters[it] = task; } // deleter tasks var deleters = new Task[DELETER_CNT]; for (int it = 0; it < DELETER_CNT; it++) { var task = new Task(() => { for (int i = 0; i < DELETER_OP_CNT; i++) { if (data.Count == 0) { System.Threading.Thread.Yield(); continue; } var idx = ExternalRandomGenerator.Instance.NextScaledRandomInteger(0, data.Count - 1); var kvp = data.ElementAt(idx); try { ipile.Delete(kvp.Key); } catch (PileAccessViolationException) { deleteAccessViolations.AddOrUpdate(System.Threading.Thread.CurrentThread.ManagedThreadId, 1, (mid, val) => val + 1); } } }); deleters[it] = task; } foreach (var task in putters) { task.Start(); } foreach (var task in getters) { task.Start(); } foreach (var task in deleters) { task.Start(); } Task.WaitAll(putters.Concat(getters).Concat(deleters).ToArray()); foreach (var kvp in getAccessViolations) { Console.WriteLine("Get thread '{0}' {1:n0} times accessed deleted pointer", kvp.Key, kvp.Value); } foreach (var kvp in deleteAccessViolations) { Console.WriteLine("Del thread '{0}' {1:n0} times accessed deleted pointer", kvp.Key, kvp.Value); } } }
public void TestThreadSafety(int cnt, int tcount, int seconds) { using (var pile = new DefaultPile()) { // pile.SegmentSize = 64 * 1024 * 1024; pile.Start(); var data = new PilePointer[cnt]; for (var i = 0; i < cnt; i++) { data[i] = pile.Put(new byte[0], preallocateBlockSize: Memory.PTR_RAW_BYTE_SIZE + (137 * (i % 17))); } var lst = new List <Task>(); long statRead = 0; long statPut = 0; long statDelete = 0; var sw = Stopwatch.StartNew(); var sd = DateTime.UtcNow; var deleteLock = new object(); for (var i = 0; i < tcount; i++) { lst.Add(Task.Factory.StartNew(() => { while (true) { var now = DateTime.UtcNow; if ((now - sd).TotalSeconds > seconds) { break; } var it = ExternalRandomGenerator.Instance.NextScaledRandomInteger(10, 100); for (var j = 0; j < it; j++) { var pp = data[ExternalRandomGenerator.Instance.NextScaledRandomInteger(0, cnt - 1)]; Aver.IsTrue(pile.Get(pp) is byte[]); Interlocked.Increment(ref statRead); } it = ExternalRandomGenerator.Instance.NextScaledRandomInteger(10, 100); for (var j = 0; j < it; j++) { var pp = data[ExternalRandomGenerator.Instance.NextScaledRandomInteger(0, cnt - 1)]; Aver.IsTrue(pile.Put(pp, new byte[ExternalRandomGenerator.Instance.NextScaledRandomInteger(0, 3791)], link: true)); Interlocked.Increment(ref statPut); } if (ExternalRandomGenerator.Instance.NextScaledRandomInteger(0, 100) > 50 && Monitor.TryEnter(deleteLock)) { try { var newData = (PilePointer[])data.Clone(); it = ExternalRandomGenerator.Instance.NextScaledRandomInteger(0, 10 + (cnt / 2)); var toDelete = new List <PilePointer>(); for (var j = 0; j < it; j++) { var idx = ExternalRandomGenerator.Instance.NextScaledRandomInteger(0, cnt - 1); toDelete.Add(newData[idx]); newData[idx] = pile.Put(new byte[12], preallocateBlockSize: ExternalRandomGenerator.Instance.NextScaledRandomInteger(24, 1024)); } data = newData;//atomic; Thread.Sleep(1000); foreach (var pp in toDelete) { Aver.IsTrue(pile.Delete(pp)); Interlocked.Increment(ref statDelete); } } finally { Monitor.Exit(deleteLock); } } } }, TaskCreationOptions.LongRunning)); } Task.WaitAll(lst.ToArray()); var el = sw.ElapsedMilliseconds; Console.WriteLine("Read {0:n0} at {1:n0} ops/sec".Args(statRead, statRead / (el / 1000d))); Console.WriteLine("Put {0:n0} at {1:n0} ops/sec".Args(statPut, statPut / (el / 1000d))); Console.WriteLine("Deleted {0:n0} at {1:n0} ops/sec".Args(statDelete, statDelete / (el / 1000d))); for (var i = 0; i < data.Length; i++) { Aver.IsTrue(pile.Delete(data[i])); } Aver.AreEqual(0, pile.ObjectCount); Aver.AreEqual(0, pile.ObjectLinkCount); Aver.AreEqual(0, pile.AllocatedMemoryBytes); } }
public void ManyRow_PutReadDeleteRead() { using (var pile = new DefaultPile()) { pile.SegmentSize = DefaultPile.SEG_SIZE_MIN; pile.Start(); var ipile = pile as IPile; var lst = new List <KeyValuePair <PilePointer, CheckoutRow> >(); int totalPut = 0; int totalDelete = 0; var sw = Stopwatch.StartNew(); while (ipile.SegmentCount < 4) { var rowIn = CheckoutRow.MakeFake(new GDID(0, (ulong)lst.Count)); var pp = ipile.Put(rowIn); totalPut++; lst.Add(new KeyValuePair <PilePointer, CheckoutRow>(pp, rowIn)); var chance = ExternalRandomGenerator.Instance.NextRandomInteger > 1000000000; //periodically delete if (!chance) { continue; } var idx = ExternalRandomGenerator.Instance.NextScaledRandomInteger(0, lst.Count - 1); var kvp = lst[idx]; lst.RemoveAt(idx); Aver.AreObjectsEqual(kvp.Value, ipile.Get(kvp.Key)); Aver.IsTrue(ipile.Delete(kvp.Key)); totalDelete++; } var wms = sw.ElapsedMilliseconds; Console.WriteLine("Created {0:n0} in {1:n0} ms at {2:n0} ops/sec".Args(totalPut, wms, totalPut / (wms / 1000d))); Console.WriteLine("Read and deleted {0:n0} in {1:n0} ms at {2:n0} ops/sec".Args(totalDelete, wms, totalDelete / (wms / 1000d))); Console.WriteLine("Left {0:n0}".Args(lst.Count)); Console.WriteLine("Occupied bytes {0:n0}".Args(pile.AllocatedMemoryBytes)); sw.Restart(); foreach (var kvp in lst) { Aver.IsNotNull(ipile.Get(kvp.Key)); } var rms = sw.ElapsedMilliseconds; Console.WriteLine("Read {0:n0} in {1:n0} ms at {2:n0} ops/sec".Args(lst.Count, rms, lst.Count / (rms / 1000d))); Console.WriteLine("Will verify all rows..."); foreach (var kvp in lst) { var rowIn = kvp.Value; var rowOut = ipile.Get(kvp.Key) as CheckoutRow; Aver.AreObjectsEqual(rowIn, rowOut); } } }
public void Parallel_ManyMixed_PutReadDeleteRead(int tcount) { using (var pile = new DefaultPile()) { pile.SegmentSize = DefaultPile.SEG_SIZE_MIN; pile.Start(); var ipile = pile as IPile; var lst = new List <KeyValuePair <PilePointer, CheckoutRow> >(); int totalBuff = 0; long totalBuffSz = 0; int totalPut = 0; int totalDelete = 0; var sw = Stopwatch.StartNew(); var tasks = new List <Task>(); while (tasks.Count < tcount) { var task = Task.Factory.StartNew(() => { var llst = new List <KeyValuePair <PilePointer, CheckoutRow> >(); var pBuff = PilePointer.Invalid; while (ipile.SegmentCount < 64) { var chance = ExternalRandomGenerator.Instance.NextRandomInteger > 0; if (chance) { var fakeBuf = new byte[ExternalRandomGenerator.Instance.NextScaledRandomInteger(0, 128 * 1024)]; Interlocked.Increment(ref totalBuff); Interlocked.Add(ref totalBuffSz, fakeBuf.Length); if (pBuff.Valid && ExternalRandomGenerator.Instance.NextRandomInteger > 0) { ipile.Delete(pBuff); //periodically delete buffers } pBuff = ipile.Put(fakeBuf); } var rowIn = CheckoutRow.MakeFake(new GDID(0, (ulong)llst.Count)); var pp = ipile.Put(rowIn); Interlocked.Increment(ref totalPut); llst.Add(new KeyValuePair <PilePointer, CheckoutRow>(pp, rowIn)); chance = ExternalRandomGenerator.Instance.NextRandomInteger > 1000000000; //periodically delete rows if (!chance) { continue; } var idx = ExternalRandomGenerator.Instance.NextScaledRandomInteger(0, llst.Count - 1); var kvp = llst[idx]; llst.RemoveAt(idx); Aver.AreObjectsEqual(kvp.Value, ipile.Get(kvp.Key)); Aver.IsTrue(ipile.Delete(kvp.Key)); Interlocked.Increment(ref totalDelete); } lock (lst) lst.AddRange(llst); }); tasks.Add(task); } Task.WaitAll(tasks.ToArray()); var wms = sw.ElapsedMilliseconds; Console.WriteLine("Buff Created {0:n0} size {1:n0} bytes".Args(totalBuff, totalBuffSz)); Console.WriteLine("Row Created {0:n0} in {1:n0} ms at {2:n0} ops/sec".Args(totalPut, wms, totalPut / (wms / 1000d))); Console.WriteLine("Read and deleted {0:n0} in {1:n0} ms at {2:n0} ops/sec".Args(totalDelete, wms, totalDelete / (wms / 1000d))); Console.WriteLine("Left {0:n0}".Args(lst.Count)); Console.WriteLine("Occupied bytes {0:n0}".Args(pile.AllocatedMemoryBytes)); Console.WriteLine("Utilized bytes {0:n0}".Args(pile.UtilizedBytes)); Console.WriteLine("Objects {0:n0}".Args(pile.ObjectCount)); sw.Restart(); foreach (var kvp in lst) { Aver.IsNotNull(ipile.Get(kvp.Key)); } var rms = sw.ElapsedMilliseconds; Console.WriteLine("Read {0:n0} in {1:n0} ms at {2:n0} ops/sec".Args(lst.Count, rms, lst.Count / (rms / 1000d))); Console.WriteLine("Will verify all rows..."); Parallel.ForEach(lst, kvp => { var rowIn = kvp.Value; var rowOut = ipile.Get(kvp.Key) as CheckoutRow; Aver.AreObjectsEqual(rowIn, rowOut); }); } }
public static void PutGetDelete_Sequential(int fromSize, int toSize, int fromObjCount, int toObjCount) { Console.WriteLine("test will take about 1 minute"); var startTime = DateTime.Now; var objects = new Dictionary <PilePointer, byte[]>(); var objectCount = 0; var objectsSumSize = 0; using (var pile = new DefaultPile() { SegmentSize = PileCacheTestCore.SEG_SIZE }) { pile.Start(); var dtStop = DateTime.Now.AddMinutes(1); while (dtStop >= DateTime.Now) { // insert routine var insertCount = NFX.ExternalRandomGenerator.Instance.NextScaledRandomInteger(fromObjCount, toObjCount); for (int i = 0; i < insertCount; i++) { var payloadSize = NFX.ExternalRandomGenerator.Instance.NextScaledRandomInteger(fromSize, toSize); var payload = new byte[payloadSize]; payload[0] = (byte)NFX.ExternalRandomGenerator.Instance.NextScaledRandomInteger(0, 255); payload[payloadSize - 1] = (byte)NFX.ExternalRandomGenerator.Instance.NextScaledRandomInteger(0, 255); var pp = pile.Put(payload); objects.Add(pp, payload); objectCount++; objectsSumSize += payloadSize; } // get if (objectCount > 0) { var getCount = ExternalRandomGenerator.Instance.NextScaledRandomInteger(5 * fromObjCount, 5 * toObjCount); for (int i = 0; i < getCount; i++) { var objectIdx = ExternalRandomGenerator.Instance.NextScaledRandomInteger(0, objectCount - 1); var obj = objects.ElementAt(objectIdx); var objPayloadFromPile = (byte[])pile.Get(obj.Key); Assert.AreEqual(obj.Value[0], objPayloadFromPile[0]); Assert.AreEqual(obj.Value[obj.Value.Length - 1], objPayloadFromPile[obj.Value.Length - 1]); Assert.AreEqual(obj.Value.Length, objPayloadFromPile.Length); } } // delete var deleteCount = ExternalRandomGenerator.Instance.NextScaledRandomInteger(fromObjCount, toObjCount); for (int i = 0; i < deleteCount; i++) { if (objectCount == 0) { break; } var objectIdx = ExternalRandomGenerator.Instance.NextScaledRandomInteger(0, objectCount - 1); var obj = objects.ElementAt(objectIdx); Assert.IsTrue(pile.Delete(obj.Key)); objects.Remove(obj.Key); objectCount--; objectsSumSize -= obj.Value.Length; } } } }
public void VarSizes_Increasing_Random(bool isParallel, int cnt, int minSz, int maxSz, bool speed, bool rnd) { using (var pile = new DefaultPile()) { pile.Start(); pile.AllocMode = speed ? AllocationMode.FavorSpeed : AllocationMode.ReuseSpace; var sw = Stopwatch.StartNew(); var tasks = new List <Task>(); for (var t = 0; t < (isParallel? (System.Environment.ProcessorCount - 1) : 1); t++) { tasks.Add( Task.Run(() => { var dict = new Dictionary <PilePointer, dummy>(); var lst = new List <PilePointer>(); var priorOdd = PilePointer.Invalid; for (var i = 0; i < cnt; i++) { var buf = new byte[12 + minSz + ( rnd ? (NFX.ExternalRandomGenerator.Instance.NextScaledRandomInteger(0, (int)(maxSz * (i / (double)cnt)))) : (int)(maxSz * (i / (double)cnt)) ) ]; var data = new dummy { bin = buf }; data.bin.WriteBEInt32(0, ExternalRandomGenerator.Instance.NextRandomInteger); data.bin.WriteBEInt32(data.bin.Length - 4, ExternalRandomGenerator.Instance.NextRandomInteger); var ptr = pile.Put(data); Assert.IsTrue(ptr.Valid); dict.Add(ptr, data); lst.Add(ptr); if (i > cnt / 3) { if (ExternalRandomGenerator.Instance.NextRandomInteger > 0) { var ri = ExternalRandomGenerator.Instance.NextScaledRandomInteger(0, lst.Count - 1); var pp = lst[ri]; if (!pp.Valid) { continue; } Assert.IsTrue(pile.Delete(pp)); dict.Remove(pp); lst[ri] = PilePointer.Invalid; } } if (i % 1000 == 0) { Console.WriteLine("Thread{0} did {1}; allocated {2} bytes, utilized {3} bytes by {4} objects {5} bytes/obj. ", Thread.CurrentThread.ManagedThreadId, i, pile.AllocatedMemoryBytes, pile.UtilizedBytes, pile.ObjectCount, pile.UtilizedBytes / pile.ObjectCount); } } Console.WriteLine("Thread {0} Population done, now checking the buffers... {1}", Thread.CurrentThread.ManagedThreadId, DateTime.Now); foreach (var entry in dict) { Assert.IsTrue(NFX.IOMiscUtils.MemBufferEquals(entry.Value.bin, (pile.Get(entry.Key) as dummy).bin)); } Console.WriteLine("Thread {0} DONE. {1}", Thread.CurrentThread.ManagedThreadId, DateTime.Now); }) );//add } Task.WaitAll(tasks.ToArray()); var el = sw.ElapsedMilliseconds; var gt = cnt * tasks.Count; Console.WriteLine("Total objects: {0:n0} in {1:n0} ms at {2:n0} obj/sec".Args(gt, el, gt / (el / 1000d))); } }
public void DeleteSeveral_TwoTables_ByteArray(bool speed, int durationSec, int putMin, int putMax, int delFactor, int payloadSizeMin, int payloadSizeMax, bool isParallel) { using (var cache = new LocalCache(NOPApplication.Instance)) using (var pile = new DefaultPile(cache)) { cache.Pile = pile; cache.PileAllocMode = speed ? AllocationMode.FavorSpeed : AllocationMode.ReuseSpace; cache.Start(); var startTime = DateTime.UtcNow; var tasks = new List <Task>(); for (var t = 0; t < (isParallel ? (System.Environment.ProcessorCount - 1) : 1); t++) { tasks.Add(Task.Factory.StartNew(() => { var list = new List <Tuple <int, GDID, int, byte, byte> >(); var tA = cache.GetOrCreateTable <GDID>("A"); var tB = cache.GetOrCreateTable <GDID>("B"); ulong k = 0; var wlc = 0; while (true) { if ((DateTime.UtcNow - startTime).TotalSeconds >= durationSec) { break; } var putCount = Ambient.Random.NextScaledRandomInteger(putMin, putMax); for (int i = 0; i < putCount; i++) { var payloadSize = Ambient.Random.NextScaledRandomInteger(payloadSizeMin, payloadSizeMax); var val = new byte[payloadSize]; val[0] = (byte)Ambient.Random.NextRandomInteger; val[payloadSize - 1] = (byte)Ambient.Random.NextRandomInteger; var tableId = Ambient.Random.NextScaledRandomInteger(0, 1); var table = tableId == 0 ? tA : tB; var key = new GDID((uint)Thread.CurrentThread.ManagedThreadId, k); table.Put(key, val); list.Add(new Tuple <int, GDID, int, byte, byte>(tableId, key, payloadSize - 1, val[0], val[payloadSize - 1])); k++; } int delCount = putCount / delFactor; for (int i = 0; i < delCount; i++) { while (true && list.Count > 0) { var idx = Ambient.Random.NextScaledRandomInteger(0, list.Count - 1); var element = list[idx]; var table = element.Item1 == 0 ? tA : tB; var key = element.Item2; var removed = table.Remove(key); list.RemoveAt(idx); if (removed) { break; } } } // get several random elements if (list.Count > 64 && Ambient.Random.NextScaledRandomInteger(0, 100) > 98) { var toRead = Ambient.Random.NextScaledRandomInteger(8, 64); wlc++; if (wlc % 125 == 0) { Console.WriteLine("Thread {0} is reading {1} elements" .Args(Thread.CurrentThread.ManagedThreadId, toRead)); } for (var j = 0; j < toRead && list.Count > 0; j++) { var idx = Ambient.Random.NextScaledRandomInteger(0, list.Count - 1); var element = list[idx]; var table = element.Item1 == 0 ? tA : tB; var key = element.Item2; var buf = table.Get(key) as byte[]; if (buf == null) { list.RemoveAt(idx); continue; } Aver.AreEqual(element.Item4, buf[0]); Aver.AreEqual(element.Item5, buf[element.Item3]); } } if (list.Count == Int32.MaxValue) { list = new List <Tuple <int, GDID, int, byte, byte> >(); } } // total check Console.WriteLine("Thread {0} is doing final read of {1} elements".Args(Thread.CurrentThread.ManagedThreadId, list.Count)); foreach (var element in list) { var table = element.Item1 == 0 ? tA : tB; var val = table.Get(element.Item2) as byte[]; if (val == null) { continue; } Aver.AreEqual(element.Item4, val[0]); Aver.AreEqual(element.Item5, val[element.Item3]); } return; }, TaskCreationOptions.LongRunning)); } Task.WaitAll(tasks.ToArray()); } }
public void Configuration() { var conf = @" app { memory-management { pile { alloc-mode=favorspeed free-list-size=100000 max-segment-limit=79 segment-size=395313143 //will be rounded to 16 byte boundary: 395,313,152 max-memory-limit=123666333000 free-chunk-sizes='128, 256, 512, 1024, 2000, 3000, 4000, 5000, 6000, 7000, 8000, 9000, 16000, 32000, 64000, 256000' } pile { name='specialNamed' free-list-size=99000 max-segment-limit=73 segment-size=395313147 //will be rounded to 16 byte boundary: 395,313,152 max-memory-limit=127666333000 free-chunk-sizes='77, 124, 180, 190, 200, 210, 220, 230, 1000, 2000, 3000, 4000, 5000, 32000, 64000, 257000' } } }".AsLaconicConfig(handling: ConvertErrorHandling.Throw); using (var app = new ServiceBaseApplication(null, conf)) { using (var pile = new DefaultPile()) { pile.Configure(null); Assert.AreEqual(AllocationMode.FavorSpeed, pile.AllocMode); Assert.AreEqual(100000, pile.FreeListSize); Assert.AreEqual(79, pile.MaxSegmentLimit); Assert.AreEqual(395313152, pile.SegmentSize); Assert.AreEqual(123666333000, pile.MaxMemoryLimit); Assert.AreEqual(128, pile.FreeChunkSizes[00]); Assert.AreEqual(256, pile.FreeChunkSizes[01]); Assert.AreEqual(512, pile.FreeChunkSizes[02]); Assert.AreEqual(1024, pile.FreeChunkSizes[03]); Assert.AreEqual(2000, pile.FreeChunkSizes[04]); Assert.AreEqual(3000, pile.FreeChunkSizes[05]); Assert.AreEqual(4000, pile.FreeChunkSizes[06]); Assert.AreEqual(5000, pile.FreeChunkSizes[07]); Assert.AreEqual(6000, pile.FreeChunkSizes[08]); Assert.AreEqual(7000, pile.FreeChunkSizes[09]); Assert.AreEqual(8000, pile.FreeChunkSizes[10]); Assert.AreEqual(9000, pile.FreeChunkSizes[11]); Assert.AreEqual(16000, pile.FreeChunkSizes[12]); Assert.AreEqual(32000, pile.FreeChunkSizes[13]); Assert.AreEqual(64000, pile.FreeChunkSizes[14]); Assert.AreEqual(256000, pile.FreeChunkSizes[15]); pile.Start();//just to test that it starts ok } using (var pile = new DefaultPile("specialNamed")) { pile.Configure(null); Assert.AreEqual(AllocationMode.ReuseSpace, pile.AllocMode); Assert.AreEqual(99000, pile.FreeListSize); Assert.AreEqual(73, pile.MaxSegmentLimit); Assert.AreEqual(395313152, pile.SegmentSize); Assert.AreEqual(127666333000, pile.MaxMemoryLimit); Assert.AreEqual(77, pile.FreeChunkSizes[00]); Assert.AreEqual(124, pile.FreeChunkSizes[01]); Assert.AreEqual(180, pile.FreeChunkSizes[02]); Assert.AreEqual(190, pile.FreeChunkSizes[03]); Assert.AreEqual(200, pile.FreeChunkSizes[04]); Assert.AreEqual(210, pile.FreeChunkSizes[05]); Assert.AreEqual(220, pile.FreeChunkSizes[06]); Assert.AreEqual(230, pile.FreeChunkSizes[07]); Assert.AreEqual(1000, pile.FreeChunkSizes[08]); Assert.AreEqual(2000, pile.FreeChunkSizes[09]); Assert.AreEqual(3000, pile.FreeChunkSizes[10]); Assert.AreEqual(4000, pile.FreeChunkSizes[11]); Assert.AreEqual(5000, pile.FreeChunkSizes[12]); Assert.AreEqual(32000, pile.FreeChunkSizes[13]); Assert.AreEqual(64000, pile.FreeChunkSizes[14]); Assert.AreEqual(257000, pile.FreeChunkSizes[15]); pile.Start();//just to test that it starts ok } }//using app }
public void Write_Read_Index_Primitives(bool usecache, string compress, string encrypt, int CNT, string idxCompress, string idxEncrypt, int pageLboundKb, int pageUboundKb, int idxLboundKb, int idxUboundKb) { var ctlMumbo = MumboJumbo.GetControl(); IPileImplementation pile = null; ICacheImplementation cache = null; IPageCache pageCache = null; if (usecache) { pile = new DefaultPile(App); cache = new LocalCache(App) { Pile = pile, DefaultTableOptions = new TableOptions("*") { CollisionMode = CollisionMode.Durable }, PileMaxMemoryLimit = 32L * 1024 * 1024 * 1024 }; pile.Start(); cache.Start(); pageCache = new PilePageCache(cache); } try { var msData = new MemoryStream(); var msIdxId = new MemoryStream(); var msIdxCid = new MemoryStream(); var msIdxDid = new MemoryStream(); var msIdxPn = new MemoryStream(); var msIdxLt = new MemoryStream(); var msIdxLn = new MemoryStream(); var msIdxAl = new MemoryStream(); var msIdxCd = new MemoryStream(); var msIdxNt = new MemoryStream(); var msIdxAmt = new MemoryStream(); var meta = VolumeMetadataBuilder.Make("Primitive Idx", "mjumbo") .SetVersion(1, 1) .SetDescription("MumboJumbo testing") .SetCompressionScheme(compress) .SetEncryptionScheme(encrypt); var volumeData = new DefaultVolume(CryptoMan, pageCache, meta, msData); var IdxIdMetaBuilder = getSpecificBuilder(GdidIdxAppender.CONTENT_TYPE_IDX_GDID, idxCompress, idxEncrypt); var IdxCidMetaBuilder = getSpecificBuilder(GuidIdxAppender.CONTENT_TYPE_IDX_GUID, idxCompress, idxEncrypt); var IdxDidMetaBuilder = getSpecificBuilder(LongIdxAppender.CONTENT_TYPE_IDX_LONG, idxCompress, idxEncrypt); var IdxPnMetaBuilder = getSpecificBuilder(IntIdxAppender.CONTENT_TYPE_IDX_INT, idxCompress, idxEncrypt); var IdxLtMetaBuilder = getSpecificBuilder(DoubleIdxAppender.CONTENT_TYPE_IDX_DOUBLE, idxCompress, idxEncrypt); var IdxLnMetaBuilder = getSpecificBuilder(DoubleIdxAppender.CONTENT_TYPE_IDX_DOUBLE, idxCompress, idxEncrypt); var IdxAlMetaBuilder = getSpecificBuilder(DecimalIdxAppender.CONTENT_TYPE_IDX_DECIMAL, idxCompress, idxEncrypt); var IdxCdMetaBuilder = getSpecificBuilder(DateTimeIdxAppender.CONTENT_TYPE_IDX_DATETIME, idxCompress, idxEncrypt); var IdxNtMetaBuilder = getSpecificBuilder(StringIdxAppender.CONTENT_TYPE_IDX_STRING, idxCompress, idxEncrypt); var IdxAmtMetaBuilder = getSpecificBuilder(AmountIdxAppender.CONTENT_TYPE_IDX_AMOUNT, idxCompress, idxEncrypt); var volumeIdxId = new DefaultVolume(CryptoMan, IdxIdMetaBuilder, msIdxId); var volumeIdxCid = new DefaultVolume(CryptoMan, IdxCidMetaBuilder, msIdxCid); var volumeIdxDid = new DefaultVolume(CryptoMan, IdxDidMetaBuilder, msIdxDid); var volumeIdxPn = new DefaultVolume(CryptoMan, IdxPnMetaBuilder, msIdxPn); var volumeIdxLt = new DefaultVolume(CryptoMan, IdxLtMetaBuilder, msIdxLt); var volumeIdxLn = new DefaultVolume(CryptoMan, IdxLnMetaBuilder, msIdxLn); var volumeIdxAl = new DefaultVolume(CryptoMan, IdxAlMetaBuilder, msIdxAl); var volumeIdxCd = new DefaultVolume(CryptoMan, IdxCdMetaBuilder, msIdxCd); var volumeIdxNt = new DefaultVolume(CryptoMan, IdxNtMetaBuilder, msIdxNt); var volumeIdxAmt = new DefaultVolume(CryptoMan, IdxAmtMetaBuilder, msIdxAmt); volumeData.PageSizeBytes = Ambient.Random.NextScaledRandomInteger(pageLboundKb, pageUboundKb) * 1024; volumeIdxId.PageSizeBytes = Ambient.Random.NextScaledRandomInteger(idxLboundKb, idxUboundKb) * 1024; volumeIdxCid.PageSizeBytes = Ambient.Random.NextScaledRandomInteger(idxLboundKb, idxUboundKb) * 1024; volumeIdxDid.PageSizeBytes = Ambient.Random.NextScaledRandomInteger(idxLboundKb, idxUboundKb) * 1024; volumeIdxPn.PageSizeBytes = Ambient.Random.NextScaledRandomInteger(idxLboundKb, idxUboundKb) * 1024; volumeIdxLt.PageSizeBytes = Ambient.Random.NextScaledRandomInteger(idxLboundKb, idxUboundKb) * 1024; volumeIdxLn.PageSizeBytes = Ambient.Random.NextScaledRandomInteger(idxLboundKb, idxUboundKb) * 1024; volumeIdxAl.PageSizeBytes = Ambient.Random.NextScaledRandomInteger(idxLboundKb, idxUboundKb) * 1024; volumeIdxCd.PageSizeBytes = Ambient.Random.NextScaledRandomInteger(idxLboundKb, idxUboundKb) * 1024; volumeIdxNt.PageSizeBytes = Ambient.Random.NextScaledRandomInteger(idxLboundKb, idxUboundKb) * 1024; volumeIdxAmt.PageSizeBytes = Ambient.Random.NextScaledRandomInteger(idxLboundKb, idxUboundKb) * 1024; var reader = new MumboJumboArchiveReader(volumeData); var idxIdReader = new GdidIdxReader(volumeIdxId); var idxCidReader = new GuidIdxReader(volumeIdxCid); var idxDidReader = new LongIdxReader(volumeIdxDid); var idxPnReader = new IntIdxReader(volumeIdxPn); var idxLtReader = new DoubleIdxReader(volumeIdxLt); var idxLnReader = new DoubleIdxReader(volumeIdxLn); var idxAlReader = new DecimalIdxReader(volumeIdxAl); var idxCdReader = new DateTimeIdxReader(volumeIdxCd); var idxNtReader = new StringIdxReader(volumeIdxNt); var idxAmtReader = new AmountIdxReader(volumeIdxAmt); var time = Azos.Time.Timeter.StartNew(); var app = Azos.Apps.ExecutionContext.Application; using (var aIdxId = new GdidIdxAppender(volumeIdxId, NOPApplication.Instance.TimeSource, NOPApplication.Instance.AppId, "dima@zhaba")) using (var aIdxCid = new GuidIdxAppender(volumeIdxCid, NOPApplication.Instance.TimeSource, NOPApplication.Instance.AppId, "dima@zhaba")) using (var aIdxDid = new LongIdxAppender(volumeIdxDid, NOPApplication.Instance.TimeSource, NOPApplication.Instance.AppId, "dima@zhaba")) using (var aIdxPn = new IntIdxAppender(volumeIdxPn, NOPApplication.Instance.TimeSource, NOPApplication.Instance.AppId, "dima@zhaba")) using (var aIdxLt = new DoubleIdxAppender(volumeIdxLt, NOPApplication.Instance.TimeSource, NOPApplication.Instance.AppId, "dima@zhaba")) using (var aIdxLn = new DoubleIdxAppender(volumeIdxLn, NOPApplication.Instance.TimeSource, NOPApplication.Instance.AppId, "dima@zhaba")) using (var aIdxAl = new DecimalIdxAppender(volumeIdxAl, NOPApplication.Instance.TimeSource, NOPApplication.Instance.AppId, "dima@zhaba")) using (var aIdxCd = new DateTimeIdxAppender(volumeIdxCd, NOPApplication.Instance.TimeSource, NOPApplication.Instance.AppId, "dima@zhaba")) using (var aIdxNt = new StringIdxAppender(volumeIdxNt, NOPApplication.Instance.TimeSource, NOPApplication.Instance.AppId, "dima@zhaba")) using (var aIdxAmt = new AmountIdxAppender(volumeIdxAmt, NOPApplication.Instance.TimeSource, NOPApplication.Instance.AppId, "dima@zhaba")) { using (var appender = new MumboJumboArchiveAppender(volumeData, NOPApplication.Instance.TimeSource, NOPApplication.Instance.AppId, "dima@zhaba", onPageCommit: (e, b) => { aIdxId.Append(new GdidBookmark(e.ID, b)); aIdxCid.Append(new GuidBookmark(e.CorrelationId, b)); aIdxDid.Append(new LongBookmark(e.DeviceId, b)); aIdxPn.Append(new IntBookmark(e.PartNumber, b)); aIdxLt.Append(new DoubleBookmark(e.Latitude, b)); aIdxLn.Append(new DoubleBookmark(e.Longitude, b)); aIdxAl.Append(new DecimalBookmark(e.Altitude, b)); aIdxCd.Append(new DateTimeBookmark(e.CreateDate, b)); aIdxNt.Append(new StringBookmark(e.Note, b)); aIdxAmt.Append(new AmountBookmark(e.Amt, b)); })) { var messages = FakeRow.GenerateMany <MumboJumbo>(1, 1, (ulong)(CNT * .5) - 1); foreach (var m in messages) { if (!App.Active) { break; } appender.Append(m); } /////////////////////////////////////////////// appender.Append(MumboJumbo.GetControl()); //we add 1 control message in the middle /////////////////////////////////////////////// messages = FakeRow.GenerateMany <MumboJumbo>(1, 1, (ulong)(CNT * .5)); foreach (var m in messages) { if (!App.Active) { break; } appender.Append(m); } } } "Wrote {0} items, now reading".SeeArgs(CNT); if (cache != null) { "Cache has {0} items".SeeArgs(cache.Count); } var gotOne = false; // Find by GDID (ID) foreach (var idx in idxIdReader.All) { if (!App.Active) { break; } var data = reader.GetEntriesStartingAt(idx.Bookmark).First(); Aver.AreEqual(data.ID, idx.Value); if (idx.Value == ctlMumbo.ID) { //data.See(); Aver.AreEqual(ctlMumbo.ID, data.ID); gotOne = true; break; } } if (!gotOne) { Aver.Fail($"Failed to find ID by {nameof(GdidIdxReader)}"); } "Finished reading {0}".SeeArgs(nameof(GdidIdxReader)); gotOne = false; // Find by Guid (CorrelationId) foreach (var idx in idxCidReader.All) { if (!App.Active) { break; } var data = reader.GetEntriesStartingAt(idx.Bookmark).First(); Aver.AreEqual(data.CorrelationId, idx.Value); if (idx.Value == ctlMumbo.CorrelationId) { //data.See(); Aver.AreEqual(ctlMumbo.CorrelationId, data.CorrelationId); gotOne = true; break; } } if (!gotOne) { Aver.Fail($"Failed to find CorrelationId by {nameof(GuidIdxReader)}"); } "Finished reading {0}".SeeArgs(nameof(GuidIdxReader)); gotOne = false; // Find by Long (DeviceId) foreach (var idx in idxDidReader.All) { if (!App.Active) { break; } var data = reader.GetEntriesStartingAt(idx.Bookmark).First(); Aver.AreEqual(data.DeviceId, idx.Value); if (idx.Value == ctlMumbo.DeviceId) { //data.See(); Aver.AreEqual(ctlMumbo.DeviceId, data.DeviceId); gotOne = true; break; } } if (!gotOne) { Aver.Fail($"Failed to find DeviceId by {nameof(LongIdxReader)}"); } "Finished reading {0}".SeeArgs(nameof(LongIdxReader)); gotOne = false; // Find by Int (PartNumber) foreach (var idx in idxPnReader.All) { if (!App.Active) { break; } var data = reader.GetEntriesStartingAt(idx.Bookmark).First(); Aver.AreEqual(data.PartNumber, idx.Value); if (idx.Value == ctlMumbo.PartNumber) { //data.See(); Aver.AreEqual(ctlMumbo.PartNumber, data.PartNumber); gotOne = true; break; } } if (!gotOne) { Aver.Fail($"Failed to find PartNumber by {nameof(IntIdxReader)}"); } "Finished reading {0}".SeeArgs(nameof(IntIdxReader)); gotOne = false; // Find by Double (Latitude) foreach (var idx in idxLtReader.All) { if (!App.Active) { break; } var data = reader.GetEntriesStartingAt(idx.Bookmark).First(); Aver.AreEqual(data.Latitude, idx.Value); if (idx.Value == ctlMumbo.Latitude) { //data.See(); Aver.AreEqual(ctlMumbo.Latitude, data.Latitude); gotOne = true; break; } } if (!gotOne) { Aver.Fail($"Failed to find Latitude by {nameof(DoubleIdxReader)}"); } "Finished reading {0}".SeeArgs(nameof(DoubleIdxReader)); gotOne = false; // Find by Double (Longitude) foreach (var idx in idxLnReader.All) { if (!App.Active) { break; } var data = reader.GetEntriesStartingAt(idx.Bookmark).First(); Aver.AreEqual(data.Longitude, idx.Value); if (idx.Value == ctlMumbo.Longitude) { //data.See(); Aver.AreEqual(ctlMumbo.Longitude, data.Longitude); gotOne = true; break; } } if (!gotOne) { Aver.Fail($"Failed to find Longitude by {nameof(DoubleIdxReader)}"); } "Finished reading {0}".SeeArgs(nameof(DoubleIdxReader)); gotOne = false; // Find by Decimal (Altitude) foreach (var idx in idxAlReader.All) { if (!App.Active) { break; } var data = reader.GetEntriesStartingAt(idx.Bookmark).First(); Aver.AreEqual(data.Altitude, idx.Value); if (idx.Value == ctlMumbo.Altitude) { //data.See(); Aver.AreEqual(ctlMumbo.Altitude, data.Altitude); gotOne = true; break; } } if (!gotOne) { Aver.Fail($"Failed to find Altitude by {nameof(DecimalIdxReader)}"); } "Finished reading {0}".SeeArgs(nameof(DecimalIdxReader)); gotOne = false; // Find by DateTime (CreateDate) foreach (var idx in idxCdReader.All) { if (!App.Active) { break; } var data = reader.GetEntriesStartingAt(idx.Bookmark).First(); Aver.AreEqual(data.CreateDate, idx.Value); if (idx.Value == ctlMumbo.CreateDate) { //data.See(); Aver.AreEqual(ctlMumbo.CreateDate, data.CreateDate); gotOne = true; break; } } if (!gotOne) { Aver.Fail($"Failed to find CreateDate by {nameof(DateTimeIdxReader)}"); } "Finished reading {0}".SeeArgs(nameof(DateTimeIdxReader)); gotOne = false; // Find by String (Note) foreach (var idx in idxNtReader.All) { if (!App.Active) { break; } var data = reader.GetEntriesStartingAt(idx.Bookmark).First(); Aver.AreEqual(data.Note, idx.Value); if (idx.Value == ctlMumbo.Note) { //data.See(); Aver.AreEqual(ctlMumbo.Note, data.Note); gotOne = true; break; } } if (!gotOne) { Aver.Fail($"Failed to find Note by {nameof(StringIdxReader)}"); } "Finished reading {0}".SeeArgs(nameof(StringIdxReader)); gotOne = false; // Find by Amount (Amt) foreach (var idx in idxAmtReader.All) { if (!App.Active) { break; } var data = reader.GetEntriesStartingAt(idx.Bookmark).First(); Aver.AreEqual(data.Amt, idx.Value); if (idx.Value == ctlMumbo.Amt) { //data.See(); Aver.AreEqual(ctlMumbo.Amt, data.Amt); gotOne = true; break; } } if (!gotOne) { Aver.Fail($"Failed to find Amt by {nameof(AmountIdxReader)}"); } "Finished reading {0}".SeeArgs(nameof(AmountIdxReader)); //gotOne = false; time.Stop(); "Did {0:n0} in {1:n1} sec at {2:n2} ops/sec\n".SeeArgs(CNT, time.ElapsedSec, CNT / time.ElapsedSec); volumeIdxId.Dispose(); volumeIdxCid.Dispose(); volumeIdxDid.Dispose(); volumeIdxPn.Dispose(); volumeIdxLt.Dispose(); volumeIdxLn.Dispose(); volumeIdxAl.Dispose(); volumeIdxCd.Dispose(); volumeIdxNt.Dispose(); volumeIdxAmt.Dispose(); volumeData.Dispose(); } finally { DisposableObject.DisposeIfDisposableAndNull(ref pageCache); DisposableObject.DisposeIfDisposableAndNull(ref cache); DisposableObject.DisposeIfDisposableAndNull(ref pile); "CLOSED all volumes\n".See(); } }
public static void DeleteSeveral_ByteArray(bool speed, int durationSec, int putMin, int putMax, int delFactor, int payloadSizeMin, int payloadSizeMax, bool isParallel) { using (var pile = new DefaultPile(NOPApplication.Instance)) { pile.AllocMode = speed ? AllocationMode.FavorSpeed : AllocationMode.ReuseSpace; pile.Start(); var startTime = DateTime.UtcNow; var tasks = new List <Task>(); for (var t = 0; t < (isParallel ? (System.Environment.ProcessorCount - 1) : 1); t++) { tasks.Add(Task.Factory.StartNew(() => { var list = new List <CheckByteArray>(); var wlc = 0; while (true) { if ((DateTime.UtcNow - startTime).TotalSeconds >= durationSec) { break; } var putCount = Ambient.Random.NextScaledRandomInteger(putMin, putMax); for (int i = 0; i < putCount; i++) { var payloadSize = Ambient.Random.NextScaledRandomInteger(payloadSizeMin, payloadSizeMax); var val = new byte[payloadSize]; val[0] = (byte)Ambient.Random.NextRandomInteger; val[payloadSize - 1] = (byte)Ambient.Random.NextRandomInteger; var ptr = pile.Put(val); list.Add(new CheckByteArray(ptr, payloadSize - 1, val[0], val[payloadSize - 1])); } int delCount = putCount / delFactor; for (int i = 0; i < delCount; i++) { var idx = Ambient.Random.NextScaledRandomInteger(0, list.Count - 1); var ptr = list[idx].Ptr; pile.Delete(ptr); list.RemoveAt(idx); } // get several random elements if (list.Count > 64 && Ambient.Random.NextScaledRandomInteger(0, 100) > 98) { var toRead = Ambient.Random.NextScaledRandomInteger(8, 64); wlc++; if (wlc % 125 == 0) { Console.WriteLine("Thread {0} is reading {1} elements, total {2}" .Args(Thread.CurrentThread.ManagedThreadId, toRead, list.Count)); } for (var k = 0; k < toRead; k++) { var element = list[Ambient.Random.NextScaledRandomInteger(0, list.Count - 1)]; var buf = pile.Get(element.Ptr) as byte[]; Aver.AreEqual(element.FirstByte, buf[0]); Aver.AreEqual(element.LastByte, buf[element.IdxLast]); } } if (list.Count == Int32.MaxValue) { list = new List <CheckByteArray>(); } } // total check Console.WriteLine("Thread {0} is doing final read of {1} elements, objectCount {2}" .Args(Thread.CurrentThread.ManagedThreadId, list.Count, pile.ObjectCount)); foreach (var element in list) { var buf = pile.Get(element.Ptr) as byte[]; Aver.AreEqual(element.FirstByte, buf[0]); Aver.AreEqual(element.LastByte, buf[element.IdxLast]); } return; })); } Task.WaitAll(tasks.ToArray()); } }
public void T060_PileNonOwnership() { var pile = new DefaultPile(); pile.Start(); try { var cache = new LocalCache(); cache.Pile = pile; cache.Configure(null); cache.Start(); var tA = cache.GetOrCreateTable <string>("A"); tA.Put("aaa", "avalue"); tA.Put("bbb", "bvalue"); Aver.AreObjectsEqual("avalue", tA.Get("aaa")); Aver.AreEqual(2, cache.Count); Aver.AreEqual(2, pile.ObjectCount); cache.WaitForCompleteStop(); Aver.IsTrue(NFX.ServiceModel.ControlStatus.Inactive == cache.Status); Aver.IsTrue(NFX.ServiceModel.ControlStatus.Active == pile.Status); Aver.AreEqual(0, pile.ObjectCount); cache = new LocalCache(); cache.Pile = pile; cache.Configure(null); cache.Start(); var tAbc = cache.GetOrCreateTable <string>("Abc"); tAbc.Put("aaa", "avalue"); tAbc.Put("bbb", "bvalue"); tAbc.Put("ccc", "cvalue"); tAbc.Put("ddd", "cvalue"); Aver.AreEqual(4, pile.ObjectCount); var cache2 = new LocalCache(); cache2.Pile = pile; cache2.Configure(null); cache2.Start(); var t2 = cache2.GetOrCreateTable <string>("A"); t2.Put("aaa", "avalue"); t2.Put("bbb", "bvalue"); Aver.AreEqual(2, cache2.Count); Aver.AreEqual(6, pile.ObjectCount); cache.WaitForCompleteStop(); Aver.IsTrue(NFX.ServiceModel.ControlStatus.Active == pile.Status); Aver.AreEqual(2, pile.ObjectCount); cache2.WaitForCompleteStop(); Aver.IsTrue(NFX.ServiceModel.ControlStatus.Active == pile.Status); Aver.AreEqual(0, pile.ObjectCount); pile.WaitForCompleteStop(); Aver.IsTrue(NFX.ServiceModel.ControlStatus.Inactive == pile.Status); } finally { pile.Dispose(); } }
private void PileForm_Load(object sender, EventArgs e) { m_Pile = new DefaultPile(); m_Pile.Configure(null); chkSpeed_CheckedChanged(null, null); }
public static void PutGetDelete_Parallel(int fromSize, int toSize, int fromObjCount, int toObjCount, int taskCount) { Console.WriteLine("test will take about 1 minute"); int objectsPut = 0, objectsDeleted = 0;//, objectGot = 0; using (var pile = new DefaultPile(NOPApplication.Instance) { SegmentSize = SEG_SIZE }) { pile.Start(); Parallel.For(0, taskCount, _ => { var startTime = DateTime.Now; var objects = new Dictionary <PilePointer, byte[]>(); var objectCount = 0; var objectsSumSize = 0; var dtStop = DateTime.Now.AddMinutes(.5); while (dtStop >= DateTime.Now) { // insert routine var insertCount = Ambient.Random.NextScaledRandomInteger(fromObjCount, toObjCount); for (int i = 0; i < insertCount; i++) { var payloadSize = Ambient.Random.NextScaledRandomInteger(fromSize, toSize); var payload = new byte[payloadSize]; payload[0] = (byte)Ambient.Random.NextScaledRandomInteger(0, 255); payload[payloadSize - 1] = (byte)Ambient.Random.NextScaledRandomInteger(0, 255); var pp = pile.Put(payload); objects.Add(pp, payload); objectCount++; objectsSumSize += payloadSize; } Interlocked.Add(ref objectsPut, insertCount); // get if (objectCount > 0) { var getCount = Ambient.Random.NextScaledRandomInteger(5 * fromObjCount, 5 * toObjCount); for (int i = 0; i < getCount; i++) { var objectIdx = Ambient.Random.NextScaledRandomInteger(0, objectCount - 1); var obj = objects.ElementAt(objectIdx); var objPayloadFromPile = (byte[])pile.Get(obj.Key); Aver.AreEqual(obj.Value[0], objPayloadFromPile[0]); Aver.AreEqual(obj.Value[obj.Value.Length - 1], objPayloadFromPile[obj.Value.Length - 1]); Aver.AreEqual(obj.Value.Length, objPayloadFromPile.Length); } } // delete var deleteCount = Ambient.Random.NextScaledRandomInteger(fromObjCount, toObjCount); if (deleteCount > objectCount) { deleteCount = objectCount; } for (int i = 0; i < deleteCount; i++) { if (objectCount == 0) { break; } var objectIdx = Ambient.Random.NextScaledRandomInteger(0, objectCount - 1); var obj = objects.ElementAt(objectIdx); Aver.IsTrue(pile.Delete(obj.Key)); objects.Remove(obj.Key); objectCount--; objectsSumSize -= obj.Value.Length; } Interlocked.Add(ref objectsDeleted, deleteCount); } }); } Console.WriteLine("put {0:N0}, deleted {1:N0} object", objectsPut, objectsDeleted); }