Пример #1
0
        public void Parallel_PutDeleteGet_Checkerboard()
        {
            const int CNT = 1002030;//1000203;
            var tuples = new Tuple<PilePointer, Charge>[CNT];

            using (var pile = new DefaultPile())
            {
              pile.Start();

              var ipile = pile as IPile;

              Parallel.For(0, CNT, i => {
            var ch = Charge.MakeFake(new GDID(0, (ulong)i));
            var pp = ipile.Put(ch);
            tuples[i] = new Tuple<PilePointer,Charge>(pp, ch);
              });

              Assert.AreEqual(CNT, ipile.ObjectCount);

              Parallel.For(0, CNT, i => {
            if (i % 3 != 0)
              ipile.Delete(tuples[i].Item1);
              });

              Assert.AreEqual(CNT/3, ipile.ObjectCount);

              var deletedHits = new ConcurrentDictionary<int, int>();

              Parallel.For(0, CNT, i => {
            if (i % 3 != 0)
            {
              try
              {
                deletedHits.AddOrUpdate(System.Threading.Thread.CurrentThread.ManagedThreadId, 1, (threadId, val) => val + 1);
                ipile.Get(tuples[i].Item1);
                Assert.Fail("Object is deleted but its pointer doesn't throw exception!");
              }
              catch (PileAccessViolationException) {}
            }
            else
            {
              var ch = ipile.Get(tuples[i].Item1);
              Assert.AreEqual(ch, tuples[i].Item2);
            }
              });

              foreach (var kvp in deletedHits)
              {
            Console.WriteLine("Thread '{0}' {1:n0} times accessed deleted pointer", kvp.Key, kvp.Value);
              }
            }
        }
Пример #2
0
      public void Initial()
      {
        using (var pile = new DefaultPile())
        {
          var ipile = pile as IPile;

          Assert.AreEqual(0, ipile.ObjectCount);
          Assert.AreEqual(0, ipile.AllocatedMemoryBytes);
          Assert.AreEqual(0, ipile.UtilizedBytes);
          Assert.AreEqual(0, ipile.OverheadBytes);
          Assert.AreEqual(0, ipile.SegmentCount);
        }
      }
Пример #3
0
      public void PutWOStart()
      {
        using (var pile = new DefaultPile())
        {
          var ipile = pile as IPile;

          var row = Charge.MakeFake(new GDID(0, 1));

          var pp = ipile.Put(row);

          Assert.IsFalse(pp.Valid);

          Assert.AreEqual(0, ipile.ObjectCount);
          Assert.AreEqual(0, ipile.AllocatedMemoryBytes);
          Assert.AreEqual(0, ipile.UtilizedBytes);
          Assert.AreEqual(0, ipile.OverheadBytes);
          Assert.AreEqual(0, ipile.SegmentCount);
        }
      }
Пример #4
0
      public void Parallel_PutGet()
      {
        const int CNT = 1000000;
        var tuples = new Tuple<PilePointer, Charge>[CNT];

        using (var pile = new DefaultPile())
        {
          pile.Start();

          var ipile = pile as IPile;

          Parallel.For(0, CNT, i => {
            var ch = Charge.MakeFake(new GDID(0, (ulong)i));
            var pp = ipile.Put(ch);
            tuples[i] = new Tuple<PilePointer,Charge>(pp, ch);
          });

          Assert.AreEqual(CNT, ipile.ObjectCount);

          Parallel.ForEach(tuples, t => {
            Assert.AreEqual(t.Item2, ipile.Get(t.Item1));
          });
        }  
      }
Пример #5
0
      public void SegmentAddedDeleted()
      {
        const int SEG_SIZE = DefaultPile.SEG_SIZE_MIN;

        using (var pile = new DefaultPile() { SegmentSize = SEG_SIZE})
        {
          pile.Start();

          var ipile = pile as IPile;

          var pp = PilePointer.Invalid;
          for(ulong i=0; i<SEG_SIZE && ipile.SegmentCount < 2; i++)
          {
            var ch = Charge.MakeFake(new GDID(1, i));
            pp = ipile.Put(ch);
          }

          Assert.AreEqual(2, ipile.SegmentCount);

          ipile.Delete(pp);

          Assert.AreEqual(1, ipile.SegmentCount);
        }
      }
Пример #6
0
        public void PutOne()
        {
            using (var pile = new DefaultPile())
            {
              pile.Start();

              var ipile = pile as IPile;

              var row = Charge.MakeFake(new GDID(0, 1));

              var pp = ipile.Put(row);

              Assert.AreEqual(1, ipile.ObjectCount);
              Assert.AreEqual(DefaultPile.SEG_SIZE_DFLT, ipile.AllocatedMemoryBytes);
              Assert.AreEqual(1, ipile.SegmentCount);
            }
        }
Пример #7
0
        public void PutGetObject()
        {
            using (var pile = new DefaultPile())
            {
              pile.Start();
              var ipile = pile as IPile;

              var rowIn = Charge.MakeFake(new GDID(0, 1));

              var pp = ipile.Put(rowIn);

              var rowOut = ipile.Get(pp) as Charge;

              Assert.AreEqual(rowIn, rowOut);
            }
        }
Пример #8
0
        public void Chessboard_ByteArray(bool speed, int durationSec, int payloadSizeMin, int payloadSizeMax, bool isParallel)
        {
            using (var cache = new LocalCache())
              using (var pile = new DefaultPile(cache))
              {
            cache.Pile = pile;
            cache.PileAllocMode = speed ? AllocationMode.FavorSpeed : AllocationMode.ReuseSpace;
            cache.Start();

            var startTime = DateTime.UtcNow;
            var tasks = new List<Task>();
            for (var t = 0; t < (isParallel ? (System.Environment.ProcessorCount - 1) : 1); t++)
              tasks.Add(Task.Factory.StartNew(() =>
            {
              var list = new List<CheckByteArray>();
              var i = 0;
              var tA = cache.GetOrCreateTable<GDID>("A");
              var wlc = 0;
              while (true)
              {
                if ((DateTime.UtcNow - startTime).TotalSeconds >= durationSec) break;

                var payloadSize = NFX.ExternalRandomGenerator
                                    .Instance.NextScaledRandomInteger(payloadSizeMin, payloadSizeMax);
                var val = new byte[payloadSize];
                val[0] = (byte)NFX.ExternalRandomGenerator.Instance.NextRandomInteger;
                val[payloadSize - 1] = (byte)NFX.ExternalRandomGenerator.Instance.NextRandomInteger;

                var key = new GDID((uint)Thread.CurrentThread.ManagedThreadId, (ulong)i);
                tA.Put(key, val);

                var element = new CheckByteArray(key, payloadSize - 1, val[0], val[payloadSize - 1]);
                list.Add(element);

                // delete previous element
                if (list.Count > 1 && i % 2 == 0)
                {
                  key = list[list.Count - 2].Key;
                  tA.Remove(key);
                  list.RemoveAt(list.Count - 2);
                }

                // get several random elements
                if (list.Count > 64 && NFX.ExternalRandomGenerator.Instance.NextScaledRandomInteger(0, 100) > 98)
                {
                  var toRead = NFX.ExternalRandomGenerator.Instance.NextScaledRandomInteger(8, 64);
                  wlc++;
                  if (wlc % 125 == 0)
                    Console.WriteLine("Thread {0} is reading {1} elements, total {2}"
                      .Args(Thread.CurrentThread.ManagedThreadId, toRead, list.Count));
                  for (var k = 0; k < toRead && list.Count > 0; k++)
                  {
                    var idx = NFX.ExternalRandomGenerator.Instance.NextScaledRandomInteger(0, list.Count - 1);
                    element = list[idx];
                    var buf = tA.Get(element.Key) as byte[];
                    if (buf == null)
                    {
                      list.RemoveAt(idx);
                      continue;
                    }
                    Assert.AreEqual(element.FirstByte, buf[0]);
                    Assert.AreEqual(element.LastByte, buf[element.IdxLast]);
                  }
                }

                if (i == Int32.MaxValue)
                  i = 0;
                else
                  i++;

                if (list.Count == Int32.MaxValue)
                  list = new List<CheckByteArray>();
              }

              // total check
              Console.WriteLine("Thread {0} is doing final read of {1} elements, tableCount {2}"
                .Args(Thread.CurrentThread.ManagedThreadId, list.Count, tA.Count));
              foreach (var element in list)
              {
                var buf = tA.Get(element.Key) as byte[];
                if (buf == null)
                  continue;
                Assert.AreEqual(element.FirstByte, buf[0]);
                Assert.AreEqual(element.LastByte, buf[element.IdxLast]);
              }
              return;
            }, TaskCreationOptions.LongRunning));
            Task.WaitAll(tasks.ToArray());
              }
        }
Пример #9
0
      public void T060_PileNonOwnership()
      {
        var pile = new DefaultPile();
        pile.Start();
        try
        {
              var cache = new LocalCache();
              cache.Pile = pile;
              cache.Configure(null);
              cache.Start();
                
        
              var tA = cache.GetOrCreateTable<string>("A");
              tA.Put("aaa", "avalue");
              tA.Put("bbb", "bvalue");
              Assert.AreEqual("avalue", tA.Get("aaa"));

              Assert.AreEqual(2, cache.Count);
              Assert.AreEqual(2, pile.ObjectCount);

              cache.WaitForCompleteStop();
              Assert.AreEqual(NFX.ServiceModel.ControlStatus.Inactive, cache.Status);
        
              Assert.AreEqual(NFX.ServiceModel.ControlStatus.Active, pile.Status);
              Assert.AreEqual(0, pile.ObjectCount);
       
              cache = new LocalCache();
              cache.Pile = pile;
              cache.Configure(null);
              cache.Start();

              var tAbc = cache.GetOrCreateTable<string>("Abc");
              tAbc.Put("aaa", "avalue");
              tAbc.Put("bbb", "bvalue");
              tAbc.Put("ccc", "cvalue");
              tAbc.Put("ddd", "cvalue");

              Assert.AreEqual(4, pile.ObjectCount);

              var cache2 = new LocalCache();
              cache2.Pile = pile;
              cache2.Configure(null);
              cache2.Start();
                
        
              var t2 = cache2.GetOrCreateTable<string>("A");
              t2.Put("aaa", "avalue");
              t2.Put("bbb", "bvalue");
        
              Assert.AreEqual(2, cache2.Count);
              Assert.AreEqual(6, pile.ObjectCount);

              cache.WaitForCompleteStop();
              Assert.AreEqual(NFX.ServiceModel.ControlStatus.Active, pile.Status);
              Assert.AreEqual(2, pile.ObjectCount);

              cache2.WaitForCompleteStop();
              Assert.AreEqual(NFX.ServiceModel.ControlStatus.Active, pile.Status);
              Assert.AreEqual(0, pile.ObjectCount);

              pile.WaitForCompleteStop();
              Assert.AreEqual(NFX.ServiceModel.ControlStatus.Inactive, pile.Status);
        }
        finally
        {
          pile.Dispose();
        }
      }
Пример #10
0
        public void Configuration()
        {
            var conf = @"
             app
             {
               memory-management
               {
             pile
             {
               alloc-mode=favorspeed
               free-list-size=100000
               max-segment-limit=79
               segment-size=395313143 //will be rounded to 16 byte boundary: 395,313,152
               max-memory-limit=123666333000

               free-chunk-sizes='128, 256, 512, 1024, 2000, 3000, 4000, 5000, 6000, 7000, 8000, 9000, 16000, 32000, 64000,  256000'
             }

             pile
             {
               name='specialNamed'
               free-list-size=99000
               max-segment-limit=73
               segment-size=395313147 //will be rounded to 16 byte boundary: 395,313,152
               max-memory-limit=127666333000

               free-chunk-sizes='77, 124, 180, 190, 200, 210, 220, 230, 1000, 2000, 3000, 4000, 5000, 32000, 64000,  257000'
             }
               }
             }".AsLaconicConfig(handling: ConvertErrorHandling.Throw);

            using(var app = new ServiceBaseApplication(null, conf))
            {
              using (var pile = new DefaultPile())
              {
            pile.Configure(null);

            Assert.AreEqual(AllocationMode.FavorSpeed, pile.AllocMode);
            Assert.AreEqual(100000, pile.FreeListSize);
            Assert.AreEqual(79, pile.MaxSegmentLimit);
            Assert.AreEqual(395313152, pile.SegmentSize);
            Assert.AreEqual(123666333000, pile.MaxMemoryLimit);

            Assert.AreEqual(128, pile.FreeChunkSizes[00]);
            Assert.AreEqual(256, pile.FreeChunkSizes[01]);
            Assert.AreEqual(512, pile.FreeChunkSizes[02]);
            Assert.AreEqual(1024, pile.FreeChunkSizes[03]);
            Assert.AreEqual(2000, pile.FreeChunkSizes[04]);
            Assert.AreEqual(3000, pile.FreeChunkSizes[05]);
            Assert.AreEqual(4000, pile.FreeChunkSizes[06]);
            Assert.AreEqual(5000, pile.FreeChunkSizes[07]);
            Assert.AreEqual(6000, pile.FreeChunkSizes[08]);
            Assert.AreEqual(7000, pile.FreeChunkSizes[09]);
            Assert.AreEqual(8000, pile.FreeChunkSizes[10]);
            Assert.AreEqual(9000, pile.FreeChunkSizes[11]);
            Assert.AreEqual(16000, pile.FreeChunkSizes[12]);
            Assert.AreEqual(32000, pile.FreeChunkSizes[13]);
            Assert.AreEqual(64000, pile.FreeChunkSizes[14]);
            Assert.AreEqual(256000, pile.FreeChunkSizes[15]);

            pile.Start();//just to test that it starts ok
              }

              using (var pile = new DefaultPile("specialNamed"))
              {
            pile.Configure(null);

            Assert.AreEqual(AllocationMode.ReuseSpace, pile.AllocMode);
            Assert.AreEqual(99000, pile.FreeListSize);
            Assert.AreEqual(73, pile.MaxSegmentLimit);
            Assert.AreEqual(395313152, pile.SegmentSize);
            Assert.AreEqual(127666333000, pile.MaxMemoryLimit);

            Assert.AreEqual(77, pile.FreeChunkSizes[00]);
            Assert.AreEqual(124, pile.FreeChunkSizes[01]);
            Assert.AreEqual(180, pile.FreeChunkSizes[02]);
            Assert.AreEqual(190, pile.FreeChunkSizes[03]);
            Assert.AreEqual(200, pile.FreeChunkSizes[04]);
            Assert.AreEqual(210, pile.FreeChunkSizes[05]);
            Assert.AreEqual(220, pile.FreeChunkSizes[06]);
            Assert.AreEqual(230, pile.FreeChunkSizes[07]);
            Assert.AreEqual(1000, pile.FreeChunkSizes[08]);
            Assert.AreEqual(2000, pile.FreeChunkSizes[09]);
            Assert.AreEqual(3000, pile.FreeChunkSizes[10]);
            Assert.AreEqual(4000, pile.FreeChunkSizes[11]);
            Assert.AreEqual(5000, pile.FreeChunkSizes[12]);
            Assert.AreEqual(32000, pile.FreeChunkSizes[13]);
            Assert.AreEqual(64000, pile.FreeChunkSizes[14]);
            Assert.AreEqual(257000, pile.FreeChunkSizes[15]);

            pile.Start();//just to test that it starts ok

              }

            }//using app
        }
Пример #11
0
 private void PileForm_Load(object sender, EventArgs e)
 {
     m_Pile = new DefaultPile();
       m_Pile.Configure(null);
       chkSpeed_CheckedChanged(null, null);
 }
Пример #12
0
        public void DeleteExisting()
        {
            using (var pile = new DefaultPile())
            {
              pile.Start();
              var ipile = pile as IPile;

              var rowIn = Charge.MakeFake(new GDID(0, 1));

              var pp = ipile.Put(rowIn);

              ipile.Delete(pp);

              Assert.AreEqual(0, ipile.ObjectCount);
              Assert.AreEqual(0, ipile.AllocatedMemoryBytes);
              Assert.AreEqual(0, ipile.UtilizedBytes);
              Assert.AreEqual(0, ipile.OverheadBytes);
              Assert.AreEqual(0, ipile.SegmentCount);

              var rowOut = ipile.Get(pp);
            }
        }
Пример #13
0
 public void DeleteInvalid()
 {
     using (var pile = new DefaultPile())
     {
       pile.Start();
       var ipile = pile as IPile;
       ipile.Delete(PilePointer.Invalid);
     }
 }
Пример #14
0
        public void PutGetRawObject()
        {
            using (var pile = new DefaultPile())
            {
              pile.Start();
              var ipile = pile as IPile;

              var buf = new byte[]{1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0};

              var pp = ipile.Put(buf);

              byte svr;
              var buf2 = ipile.GetRawBuffer(pp, out svr); //main point: we dont get any exceptions

              Assert.IsTrue(buf2.Length >= buf.Length);
            }
        }
Пример #15
0
               public _segment(DefaultPile pile)
               {
                 Pile = pile;
                 Data = new byte[pile.m_SegmentSize];//segment memory

                 //init 1 chunk for the whole segment
                 var adr = 0;
                 writeFreeFlag(Data, 0); adr+=3;//flag
                 Data.WriteBEInt32(adr, Data.Length - CHUNK_HDER_SZ); 

                 FreeChunks = new freeChunks[FREE_LST_COUNT];
                 for(var i=0; i<FreeChunks.Length; i++)
                 {
                   var chunk = new freeChunks();
                   chunk.Addresses = new int[pile.m_FreeListSize];
                   chunk.CurrentIndex = -1;
                   FreeChunks[i] = chunk;
                 }
                 FreeChunks[FreeChunks.Length-1].CurrentIndex = 0;//mark the largest chunk as free
               }
Пример #16
0
        public static void PutGetDelete_Sequential(int fromSize, int toSize, int fromObjCount, int toObjCount)
        {
            Console.WriteLine("test will take about 1 minute");

            var startTime = DateTime.Now;
            var objects = new Dictionary<PilePointer, byte[]>();
            var objectCount = 0;
            var objectsSumSize = 0;

            using (var pile = new DefaultPile() { SegmentSize = PileCacheTestCore.SEG_SIZE })
            {
                pile.Start();

                var dtStop = DateTime.Now.AddMinutes(1);
                while (dtStop >= DateTime.Now)
                {
                    // insert routine
                    var insertCount = NFX.ExternalRandomGenerator.Instance.NextScaledRandomInteger(fromObjCount, toObjCount);
                    for (int i = 0; i < insertCount; i++)
                    {
                        var payloadSize = NFX.ExternalRandomGenerator.Instance.NextScaledRandomInteger(fromSize, toSize);
                        var payload = new byte[payloadSize];
                        payload[0] = (byte)NFX.ExternalRandomGenerator.Instance.NextScaledRandomInteger(0, 255);
                        payload[payloadSize - 1] = (byte)NFX.ExternalRandomGenerator.Instance.NextScaledRandomInteger(0, 255);
                        var pp = pile.Put(payload);
                        objects.Add(pp, payload);
                        objectCount++;
                        objectsSumSize += payloadSize;
                    }

                    // get
                    if (objectCount > 0)
                    {
                        var getCount = ExternalRandomGenerator.Instance.NextScaledRandomInteger(5 * fromObjCount, 5 * toObjCount);
                        for (int i = 0; i < getCount; i++)
                        {
                            var objectIdx = ExternalRandomGenerator.Instance.NextScaledRandomInteger(0, objectCount - 1);
                            var obj = objects.ElementAt(objectIdx);
                            var objPayloadFromPile = (byte[])pile.Get(obj.Key);
                            Assert.AreEqual(obj.Value[0], objPayloadFromPile[0]);
                            Assert.AreEqual(obj.Value[obj.Value.Length - 1], objPayloadFromPile[obj.Value.Length - 1]);
                            Assert.AreEqual(obj.Value.Length, objPayloadFromPile.Length);
                        }
                    }

                    // delete
                    var deleteCount = ExternalRandomGenerator.Instance.NextScaledRandomInteger(fromObjCount, toObjCount);
                    for (int i = 0; i < deleteCount; i++)
                    {
                        if (objectCount == 0) break;

                        var objectIdx = ExternalRandomGenerator.Instance.NextScaledRandomInteger(0, objectCount - 1);
                        var obj = objects.ElementAt(objectIdx);

                        Assert.IsTrue(pile.Delete(obj.Key));
                        objects.Remove(obj.Key);
                        objectCount--; objectsSumSize -= obj.Value.Length;
                    }
                }
            }
        }
Пример #17
0
        public void NoGrowth_ByteArray(bool speed, int durationSec, int payloadSizeMin, int payloadSizeMax, int countMin, int countMax)
        {
            using (var cache = new LocalCache())
              using (var pile = new DefaultPile(cache))
              {
            cache.Pile = pile;
            cache.PileAllocMode = speed ? AllocationMode.FavorSpeed : AllocationMode.ReuseSpace;
            cache.Start();

            var startTime = DateTime.UtcNow;
            var tasks = new List<Task>();
            for (var t = 0; t < (System.Environment.ProcessorCount - 1); t++)
              tasks.Add(Task.Factory.StartNew(() =>
            {
              var tA = cache.GetOrCreateTable<GDID>("A");
              var list = new List<CheckByteArray>();
              bool put = true;

              while (true)
              {
                if ((DateTime.UtcNow - startTime).TotalSeconds >= durationSec) return;

                if (put)
                {
                  var cnt = NFX.ExternalRandomGenerator.Instance.NextScaledRandomInteger(countMin, countMax);
                  for (int i = 0; i < cnt; i++)
                  {
                    var payloadSize = NFX.ExternalRandomGenerator
                                        .Instance.NextScaledRandomInteger(payloadSizeMin, payloadSizeMax);
                    var val = new byte[payloadSize];
                    val[0] = (byte)NFX.ExternalRandomGenerator.Instance.NextRandomInteger;
                    val[payloadSize - 1] = (byte)NFX.ExternalRandomGenerator.Instance.NextRandomInteger;
                    var key = new GDID((uint)Thread.CurrentThread.ManagedThreadId, (ulong)i);

                    tA.Put(key, val);

                    var element = new CheckByteArray(key, payloadSize - 1, val[0], val[payloadSize - 1]);
                    list.Add(element);
                  }
                  Console.WriteLine("Thread {0} put {1} objects".Args(Thread.CurrentThread.ManagedThreadId, list.Count));
                  put = false;
                }
                else
                {
                  var i = 0;
                  for (var j = 0; j < list.Count; j++)
                  {
                    var element = list[j];
                    var buf = tA.Get(element.Key) as byte[];
                    if (buf != null)
                    {
                      Assert.AreEqual(element.FirstByte, buf[0]);
                      Assert.AreEqual(element.LastByte, buf[element.IdxLast]);
                      tA.Remove(element.Key);
                      i++;
                    }
                  }
                  Console.WriteLine("Thread {0} deleted {1} objects".Args(Thread.CurrentThread.ManagedThreadId, i));
                  list.Clear();
                  put = true;
                }
              }
            }, TaskCreationOptions.LongRunning));
            Task.WaitAll(tasks.ToArray());
              }
        }
Пример #18
0
        public void DeleteSeveral_TwoTables_ByteArray(bool speed, int durationSec, int putMin, int putMax, int delFactor, int payloadSizeMin, int payloadSizeMax, bool isParallel)
        {
            using (var cache = new LocalCache())
              using (var pile = new DefaultPile(cache))
              {
            cache.Pile = pile;
            cache.PileAllocMode = speed ? AllocationMode.FavorSpeed : AllocationMode.ReuseSpace;
            cache.Start();

            var startTime = DateTime.UtcNow;
            var tasks = new List<Task>();
            for (var t = 0; t < (isParallel ? (System.Environment.ProcessorCount - 1) : 1); t++)
              tasks.Add(Task.Factory.StartNew(() =>
            {
              var list = new List<Tuple<int, GDID, int, byte, byte>>();
              var tA = cache.GetOrCreateTable<GDID>("A");
              var tB = cache.GetOrCreateTable<GDID>("B");
              ulong k = 0;
              var wlc = 0;

              while (true)
              {
                if ((DateTime.UtcNow - startTime).TotalSeconds >= durationSec) break;

                var putCount = NFX.ExternalRandomGenerator.Instance.NextScaledRandomInteger(putMin, putMax);
                for (int i = 0; i < putCount; i++)
                {
                  var payloadSize = NFX.ExternalRandomGenerator
                                      .Instance.NextScaledRandomInteger(payloadSizeMin, payloadSizeMax);
                  var val = new byte[payloadSize];
                  val[0] = (byte)NFX.ExternalRandomGenerator.Instance.NextRandomInteger;
                  val[payloadSize - 1] = (byte)NFX.ExternalRandomGenerator.Instance.NextRandomInteger;

                  var tableId = NFX.ExternalRandomGenerator.Instance.NextScaledRandomInteger(0, 1);
                  var table = tableId == 0 ? tA : tB;
                  var key = new GDID((uint)Thread.CurrentThread.ManagedThreadId, k);

                  table.Put(key, val);

                  list.Add(new Tuple<int, GDID, int, byte, byte>(tableId, key, payloadSize - 1, val[0], val[payloadSize - 1]));
                  k++;
                }

                int delCount = putCount / delFactor;
                for (int i = 0; i < delCount; i++)
                {
                  while (true && list.Count > 0)
                  {
                    var idx = NFX.ExternalRandomGenerator.Instance.NextScaledRandomInteger(0, list.Count - 1);
                    var element = list[idx];
                    var table = element.Item1 == 0 ? tA : tB;
                    var key = element.Item2;

                    var removed = table.Remove(key);
                    list.RemoveAt(idx);
                    if (removed)
                      break;
                  }
                }

                // get several random elements
                if (list.Count > 64 && NFX.ExternalRandomGenerator.Instance.NextScaledRandomInteger(0, 100) > 98)
                {
                  var toRead = NFX.ExternalRandomGenerator.Instance.NextScaledRandomInteger(8, 64);
                  wlc++;
                  if (wlc % 125 == 0)
                    Console.WriteLine("Thread {0} is reading {1} elements"
                      .Args(Thread.CurrentThread.ManagedThreadId, toRead));
                  for (var j = 0; j < toRead && list.Count > 0; j++)
                  {
                    var idx = NFX.ExternalRandomGenerator.Instance.NextScaledRandomInteger(0, list.Count - 1);
                    var element = list[idx];
                    var table = element.Item1 == 0 ? tA : tB;
                    var key = element.Item2;

                    var buf = table.Get(key) as byte[];
                    if (buf == null)
                    {
                      list.RemoveAt(idx);
                      continue;
                    }
                    Assert.AreEqual(element.Item4, buf[0]);
                    Assert.AreEqual(element.Item5, buf[element.Item3]);
                  }
                }

                if (list.Count == Int32.MaxValue)
                  list = new List<Tuple<int, GDID, int, byte, byte>>();
              }

              // total check
              Console.WriteLine("Thread {0} is doing final read of {1} elements".Args(Thread.CurrentThread.ManagedThreadId, list.Count));
              foreach (var element in list)
              {
                var table = element.Item1 == 0 ? tA : tB;
                var val = table.Get(element.Item2) as byte[];
                if (val == null)
                  continue;
                Assert.AreEqual(element.Item4, val[0]);
                Assert.AreEqual(element.Item5, val[element.Item3]);
              }
              return;
            }, TaskCreationOptions.LongRunning));
            Task.WaitAll(tasks.ToArray());
              }
        }
Пример #19
0
        public void PutTwo()
        {
            using (var pile = new DefaultPile())
            {
              pile.Start();

              var ipile = pile as IPile;

              var rowIn1 = Charge.MakeFake(new GDID(0, 1));
              var rowIn2 = Charge.MakeFake(new GDID(0, 2));

              var pp1 = ipile.Put(rowIn1);
              var pp2 = ipile.Put(rowIn2);

              Assert.AreEqual(2, ipile.ObjectCount);
              Assert.AreEqual(DefaultPile.SEG_SIZE_DFLT, ipile.AllocatedMemoryBytes);
              Assert.AreEqual(1, ipile.SegmentCount);

              var rowOut1 = pile.Get(pp1) as Charge;
              var rowOut2 = pile.Get(pp2) as Charge;

              Assert.AreEqual(rowIn1, rowOut1);
              Assert.AreEqual(rowIn2, rowOut2);
            }
        }
Пример #20
0
 public void GetNoObject()
 {
     using (var pile = new DefaultPile())
     {
       pile.Start();
       var ipile = pile as IPile;
       ipile.Get(PilePointer.Invalid);
     }
 }
Пример #21
0
        public void VarSizes_Checkboard(bool isParallel, int cnt, int minSz, int maxSz, bool speed)
        {
            using (var pile = new DefaultPile())
            {
              pile.Start();
              pile.AllocMode = speed ? AllocationMode.FavorSpeed : AllocationMode.ReuseSpace;

              var tasks = new List<Task>();
              for(var t=0; t < (isParallel? (System.Environment.ProcessorCount - 1) : 1); t++)
               tasks.Add(
                  Task.Run( () =>
                  {
                     var dict = new Dictionary<PilePointer, dummy>();
                     var priorOdd = PilePointer.Invalid;
                     for(var i=0; i<cnt; i++)
                     {
                       var even = (i&0x01)==0;
                       var data = new dummy{ bin = new byte[12 + NFX.ExternalRandomGenerator.Instance.NextScaledRandomInteger(minSz, maxSz)]};
                       data.bin.WriteBEInt32(0, ExternalRandomGenerator.Instance.NextRandomInteger);
                       data.bin.WriteBEInt32(data.bin.Length-4, ExternalRandomGenerator.Instance.NextRandomInteger);
                       var ptr = pile.Put(data);
                       Assert.IsTrue( ptr.Valid );

                       if (even)
                        dict.Add(ptr, data);
                       else
                       {
                        if (priorOdd.Valid)
                         Assert.IsTrue( pile.Delete(priorOdd) );
                        priorOdd = ptr;
                       }

                       if (i%1000==0)
                        Console.WriteLine("Thread{0} did {1}; allocated {2} bytes, utilized {3} bytes by {4} objects {5} bytes/obj. ",
                                          Thread.CurrentThread.ManagedThreadId,
                                           i,
                                           pile.AllocatedMemoryBytes,
                                           pile.UtilizedBytes,
                                           pile.ObjectCount,
                                           pile.UtilizedBytes / pile.ObjectCount);

                     }
                     Console.WriteLine("Thread {0} Population done, now checking the buffers... {1}",Thread.CurrentThread.ManagedThreadId, DateTime.Now);

                     foreach(var entry in dict)
                       Assert.IsTrue( NFX.IOMiscUtils.MemBufferEquals(entry.Value.bin, (pile.Get(entry.Key) as dummy).bin));

                     Console.WriteLine("Thread {0} DONE. {1}", Thread.CurrentThread.ManagedThreadId, DateTime.Now);
                  })
            );//add

              Task.WaitAll( tasks.ToArray() );
            }
        }
Пример #22
0
 public void GetWOStart()
 {
     using (var pile = new DefaultPile())
     {
       var ipile = pile as IPile;
       var obj = ipile.Get(PilePointer.Invalid);
       Assert.IsNull(obj);
     }
 }
Пример #23
0
        public void VarSizes_Increasing_Random(bool isParallel, int cnt, int minSz, int maxSz, bool speed, bool rnd)
        {
            using (var pile = new DefaultPile())
            {
              pile.Start();
              pile.AllocMode = speed ? AllocationMode.FavorSpeed : AllocationMode.ReuseSpace;

              var sw = Stopwatch.StartNew();
              var tasks = new List<Task>();
              for(var t=0; t < (isParallel? (System.Environment.ProcessorCount - 1) : 1); t++)
               tasks.Add(
                  Task.Run( () =>
                  {
                     var dict = new Dictionary<PilePointer, dummy>();
                     var lst = new List<PilePointer>();
                     var priorOdd = PilePointer.Invalid;
                     for(var i=0; i<cnt; i++)
                     {
                       var buf = new byte[12 +
                                          minSz+
                                          (
                                          rnd
                                           ? (NFX.ExternalRandomGenerator.Instance.NextScaledRandomInteger(0, (int)(maxSz*(i/(double)cnt))))
                                           : (int)(maxSz*(i/(double)cnt))
                                          )
                                         ];
                       var data = new dummy{ bin = buf};
                       data.bin.WriteBEInt32(0, ExternalRandomGenerator.Instance.NextRandomInteger);
                       data.bin.WriteBEInt32(data.bin.Length-4, ExternalRandomGenerator.Instance.NextRandomInteger);
                       var ptr = pile.Put(data);
                       Assert.IsTrue( ptr.Valid );

                       dict.Add(ptr, data);
                       lst.Add(ptr);

                       if (i>cnt/3)
                       {
                         if (ExternalRandomGenerator.Instance.NextRandomInteger > 0)
                         {
                           var ri = ExternalRandomGenerator.Instance.NextScaledRandomInteger(0, lst.Count-1);
                           var pp = lst[ri];
                           if (!pp.Valid) continue;

                           Assert.IsTrue( pile.Delete(pp) );
                           dict.Remove( pp );
                           lst[ri] = PilePointer.Invalid;

                         }
                       }

                       if (i%1000==0)
                        Console.WriteLine("Thread{0} did {1}; allocated {2} bytes, utilized {3} bytes by {4} objects {5} bytes/obj. ",
                                          Thread.CurrentThread.ManagedThreadId,
                                           i,
                                           pile.AllocatedMemoryBytes,
                                           pile.UtilizedBytes,
                                           pile.ObjectCount,
                                           pile.UtilizedBytes / pile.ObjectCount);

                     }
                     Console.WriteLine("Thread {0} Population done, now checking the buffers... {1}",Thread.CurrentThread.ManagedThreadId, DateTime.Now);

                     foreach(var entry in dict)
                       Assert.IsTrue( NFX.IOMiscUtils.MemBufferEquals(entry.Value.bin, (pile.Get(entry.Key) as dummy).bin));

                     Console.WriteLine("Thread {0} DONE. {1}", Thread.CurrentThread.ManagedThreadId, DateTime.Now);
                  })
            );//add

              Task.WaitAll( tasks.ToArray() );
              var el = sw.ElapsedMilliseconds;
              var gt = cnt * tasks.Count;
              Console.WriteLine("Total objects: {0:n0} in {1:n0} ms at {2:n0} obj/sec".Args(gt, el, gt / (el / 1000d)));
            }
        }
Пример #24
0
      public void PileDeleteInLastSegment(int payloadSize)
      {
        using (var pile = new DefaultPile() { SegmentSize = SEG_SIZE})
        {
          pile.Start();

          var pps = new List<PilePointer>();
          while (pile.SegmentCount < 2)
            pps.Add(pile.Put(generatePayload(payloadSize)));

          pile.Delete(pps.Last());
          pps.RemoveAt(pps.Count-1);

          Console.WriteLine("segment count: {0}, segment total count: {1}", pile.SegmentCount, pile.SegmentTotalCount);

          Assert.AreEqual(1, pile.SegmentCount);
          Assert.AreEqual(1, pile.SegmentTotalCount);

        }   
      }
Пример #25
0
      public void T050_PileNonOwnershipErrorStart()
      {
        var pile = new DefaultPile();

        try
        {
          var cache = new LocalCache();
          cache.Pile = pile;
          cache.Configure(null);
          cache.Start();  //can not start cache that uses inactive pile which is not managed by this cache
        }
        finally
        {
          pile.Dispose();
        }
                
      }
Пример #26
0
        public void PutCheckerboardPattern2()
        {
            using (var pile = new DefaultPile())
            {
              pile.Start();
              var ipile = pile as IPile;

              const ulong CNT = 100;

              var ppp = new Tuple<PilePointer, Charge>[CNT];

              for (ulong i = 0; i < CNT; i++)
              {
            var ch = Charge.MakeFake(new GDID(0, i));
            ppp[i] = new Tuple<PilePointer,Charge>( ipile.Put(ch), ch);
              }

              Assert.AreEqual(CNT, ipile.ObjectCount);

              for(ulong i = 0; i < CNT; i++)
              {
            var ch = ipile.Get(ppp[i].Item1);
            Assert.AreEqual(ch, ppp[i].Item2);
              }

              for(ulong i = 0; i < CNT; i+=2)
            ipile.Delete(ppp[i].Item1);

              Assert.AreEqual(CNT/2, ipile.ObjectCount);

              for(ulong i = 0; i < CNT; i++)
              {
            if (i % 2 == 0)
            {
              try
              {
                ipile.Get(ppp[i].Item1);
                Assert.Fail("Object is deleted but its pointer doesn't throw exception!");
              }
              catch (PileAccessViolationException) {}
            }
            else
            {
              var ch = ipile.Get(ppp[i].Item1);
              Assert.AreEqual(ch, ppp[i].Item2);
            }
              }
            }
        }
Пример #27
0
      public void Parallel_PutGetDelete_Random()
      {
        const int PUTTER_CNT  = 2, PUTTER_OP_CNT  = 2 * 10000;
        const int GETTER_CNT  = 6, GETTER_OP_CNT  = 2 * 30000;
        const int DELETER_CNT = 2, DELETER_OP_CNT = 2 * 10000;

        var data = new ConcurrentDictionary<PilePointer, string>();

        var getAccessViolations = new ConcurrentDictionary<int, int>();
        var deleteAccessViolations = new ConcurrentDictionary<int, int>();

        using (var pile = new DefaultPile())
        {
          pile.Start();

          var ipile = pile as IPile;

          // putter tasks
          var putters = new Task[PUTTER_CNT];
          for (int it = 0; it < PUTTER_CNT; it++)
          {
            var task = new Task(() => {

              for (int i = 0; i < PUTTER_OP_CNT; i++)
              {
                var str = NFX.Parsing.NaturalTextGenerator.Generate();
                var pp = ipile.Put(str);
                data.TryAdd(pp, str);
              }

            });

            putters[it] = task;
          }

          // getter tasks
          var getters = new Task[GETTER_CNT];
          for (int it = 0; it < GETTER_CNT; it++)
          {
            var task = new Task(() => {

              for (int i = 0; i < GETTER_OP_CNT; i++)
              {
                if (data.Count == 0) {
                  System.Threading.Thread.Yield();
                  continue;
                }
                var idx = ExternalRandomGenerator.Instance.NextScaledRandomInteger(0, data.Count-1);
                var kvp = data.ElementAt(idx);
                try
                {
                  
                  var str = ipile.Get(kvp.Key);
                  Assert.AreEqual(str, kvp.Value);
                }
                catch (PileAccessViolationException)
                {
                  getAccessViolations.AddOrUpdate(System.Threading.Thread.CurrentThread.ManagedThreadId, 1, (mid, val) => val + 1);
                }
              }
            });
            getters[it] = task;
          }

          // deleter tasks
          var deleters = new Task[DELETER_CNT];
          for (int it = 0; it < DELETER_CNT; it++)
          {
            var task = new Task(() => {

              for (int i = 0; i < DELETER_OP_CNT; i++)
              {
                if (data.Count == 0) {
                  System.Threading.Thread.Yield();
                  continue;
                }
                var idx = ExternalRandomGenerator.Instance.NextScaledRandomInteger(0, data.Count-1);
                var kvp = data.ElementAt(idx);
                try
                {
                  ipile.Delete(kvp.Key);
                }
                catch (PileAccessViolationException)
                {
                  deleteAccessViolations.AddOrUpdate(System.Threading.Thread.CurrentThread.ManagedThreadId, 1, (mid, val) => val + 1);
                }
              }
            });
            deleters[it] = task;
          }


          foreach (var task in putters) task.Start();
          foreach (var task in getters) task.Start();
          foreach (var task in deleters) task.Start();


          Task.WaitAll(putters.Concat(getters).Concat(deleters).ToArray());

          foreach (var kvp in getAccessViolations)
            Console.WriteLine("Get thread '{0}' {1:n0} times accessed deleted pointer", kvp.Key, kvp.Value);

          foreach (var kvp in deleteAccessViolations)
            Console.WriteLine("Del thread '{0}' {1:n0} times accessed deleted pointer", kvp.Key, kvp.Value);
        }
      }
Пример #28
0
        public void PutCheckerboardPattern3()
        {
            using (var pile = new DefaultPile())
            {
              pile.Start();
              var ipile = pile as IPile;

              const ulong CNT = 123;

              var ppp = new Tuple<PilePointer, string>[CNT];

              for (ulong i = 0; i < CNT; i++)
              {
            var str = NFX.Parsing.NaturalTextGenerator.Generate(179);
            ppp[i] = new Tuple<PilePointer,string>( ipile.Put(str), str);
              }

              Assert.AreEqual(CNT, ipile.ObjectCount);

              for(ulong i = 0; i < CNT; i++)
              {
            if (i % 3 != 0)
              ipile.Delete(ppp[i].Item1);
              }

              Assert.AreEqual(CNT/3, ipile.ObjectCount);

              for(ulong i = 0; i < CNT; i++)
              {
            if (i % 3 != 0)
            {
              try
              {
                ipile.Get(ppp[i].Item1);
                Assert.Fail("Object is deleted but its pointer doesn't throw exception!");
              }
              catch (PileAccessViolationException) {}
            }
            else
            {
              var ch = ipile.Get(ppp[i].Item1);
              Assert.AreEqual(ch, ppp[i].Item2);
            }
              }

              ////Console.WriteLine("ObjectCount: {0}", ipile.ObjectCount);
              ////Console.WriteLine("AllocatedMemoryBytes: {0}", ipile.AllocatedMemoryBytes);
              ////Console.WriteLine("UtilizedBytes: {0}", ipile.UtilizedBytes);
              ////Console.WriteLine("OverheadBytes: {0}", ipile.OverheadBytes);
              ////Console.WriteLine("SegmentCount: {0}", ipile.SegmentCount);
            }
        }
Пример #29
0
        public void Purge()
        {
            using (var pile = new DefaultPile())
            {
              pile.Start();
              var ipile = pile as IPile;

              var rowIn = Charge.MakeFake(new GDID(0, 1));

              var pp = ipile.Put(rowIn);

              ipile.Purge();

              Assert.AreEqual(0, ipile.ObjectCount);
              Assert.AreEqual(0, ipile.SegmentCount);

              var rowOut = ipile.Get(pp);

            }
        }
Пример #30
0
      public void PileDeleteInMiddleSegment(int payloadSize)
      {
        using (var pile = new DefaultPile() { SegmentSize = SEG_SIZE})
        {
          pile.Start();

          var pps = new List<PilePointer>();
          while (pile.SegmentCount < 4)
            pps.Add(pile.Put(generatePayload(payloadSize)));

          pile.Delete(pps.Last());
          pps.RemoveAt(pps.Count-1);

          var objectsInsegmentCount = pps.Count / 3;

          Console.WriteLine("{0:N0} object in pile, {1:N0} object per segment", pile.ObjectCount, objectsInsegmentCount);

          Assert.AreEqual(3, pile.SegmentCount);
          Assert.AreEqual(3, pile.SegmentTotalCount);

          for (int i = objectsInsegmentCount; i < 2 * objectsInsegmentCount; i++)
          {
            pile.Delete(pps[i]);
          }

          Console.WriteLine("{0:N0} object in pile, {1:N0} segments, {2:N0} segments total", pile.ObjectCount, pile.SegmentCount, pile.SegmentTotalCount);

          Assert.AreEqual(2, pile.SegmentCount);
          Assert.AreEqual(3, pile.SegmentTotalCount);
        }   
      }