Exemplo n.º 1
0
        public void VarSizes_Increasing_Random(bool isParallel, int cnt, int minSz, int maxSz, bool speed, bool rnd)
        {
            using (var pile = new DefaultPile())
            {
              pile.Start();
              pile.AllocMode = speed ? AllocationMode.FavorSpeed : AllocationMode.ReuseSpace;

              var sw = Stopwatch.StartNew();
              var tasks = new List<Task>();
              for(var t=0; t < (isParallel? (System.Environment.ProcessorCount - 1) : 1); t++)
               tasks.Add(
                  Task.Run( () =>
                  {
                     var dict = new Dictionary<PilePointer, dummy>();
                     var lst = new List<PilePointer>();
                     var priorOdd = PilePointer.Invalid;
                     for(var i=0; i<cnt; i++)
                     {
                       var buf = new byte[12 +
                                          minSz+
                                          (
                                          rnd
                                           ? (NFX.ExternalRandomGenerator.Instance.NextScaledRandomInteger(0, (int)(maxSz*(i/(double)cnt))))
                                           : (int)(maxSz*(i/(double)cnt))
                                          )
                                         ];
                       var data = new dummy{ bin = buf};
                       data.bin.WriteBEInt32(0, ExternalRandomGenerator.Instance.NextRandomInteger);
                       data.bin.WriteBEInt32(data.bin.Length-4, ExternalRandomGenerator.Instance.NextRandomInteger);
                       var ptr = pile.Put(data);
                       Assert.IsTrue( ptr.Valid );

                       dict.Add(ptr, data);
                       lst.Add(ptr);

                       if (i>cnt/3)
                       {
                         if (ExternalRandomGenerator.Instance.NextRandomInteger > 0)
                         {
                           var ri = ExternalRandomGenerator.Instance.NextScaledRandomInteger(0, lst.Count-1);
                           var pp = lst[ri];
                           if (!pp.Valid) continue;

                           Assert.IsTrue( pile.Delete(pp) );
                           dict.Remove( pp );
                           lst[ri] = PilePointer.Invalid;

                         }
                       }

                       if (i%1000==0)
                        Console.WriteLine("Thread{0} did {1}; allocated {2} bytes, utilized {3} bytes by {4} objects {5} bytes/obj. ",
                                          Thread.CurrentThread.ManagedThreadId,
                                           i,
                                           pile.AllocatedMemoryBytes,
                                           pile.UtilizedBytes,
                                           pile.ObjectCount,
                                           pile.UtilizedBytes / pile.ObjectCount);

                     }
                     Console.WriteLine("Thread {0} Population done, now checking the buffers... {1}",Thread.CurrentThread.ManagedThreadId, DateTime.Now);

                     foreach(var entry in dict)
                       Assert.IsTrue( NFX.IOMiscUtils.MemBufferEquals(entry.Value.bin, (pile.Get(entry.Key) as dummy).bin));

                     Console.WriteLine("Thread {0} DONE. {1}", Thread.CurrentThread.ManagedThreadId, DateTime.Now);
                  })
            );//add

              Task.WaitAll( tasks.ToArray() );
              var el = sw.ElapsedMilliseconds;
              var gt = cnt * tasks.Count;
              Console.WriteLine("Total objects: {0:n0} in {1:n0} ms at {2:n0} obj/sec".Args(gt, el, gt / (el / 1000d)));
            }
        }
Exemplo n.º 2
0
        public static void PutGetDelete_Sequential(int fromSize, int toSize, int fromObjCount, int toObjCount)
        {
            Console.WriteLine("test will take about 1 minute");

            var startTime = DateTime.Now;
            var objects = new Dictionary<PilePointer, byte[]>();
            var objectCount = 0;
            var objectsSumSize = 0;

            using (var pile = new DefaultPile() { SegmentSize = PileCacheTestCore.SEG_SIZE })
            {
                pile.Start();

                var dtStop = DateTime.Now.AddMinutes(1);
                while (dtStop >= DateTime.Now)
                {
                    // insert routine
                    var insertCount = NFX.ExternalRandomGenerator.Instance.NextScaledRandomInteger(fromObjCount, toObjCount);
                    for (int i = 0; i < insertCount; i++)
                    {
                        var payloadSize = NFX.ExternalRandomGenerator.Instance.NextScaledRandomInteger(fromSize, toSize);
                        var payload = new byte[payloadSize];
                        payload[0] = (byte)NFX.ExternalRandomGenerator.Instance.NextScaledRandomInteger(0, 255);
                        payload[payloadSize - 1] = (byte)NFX.ExternalRandomGenerator.Instance.NextScaledRandomInteger(0, 255);
                        var pp = pile.Put(payload);
                        objects.Add(pp, payload);
                        objectCount++;
                        objectsSumSize += payloadSize;
                    }

                    // get
                    if (objectCount > 0)
                    {
                        var getCount = ExternalRandomGenerator.Instance.NextScaledRandomInteger(5 * fromObjCount, 5 * toObjCount);
                        for (int i = 0; i < getCount; i++)
                        {
                            var objectIdx = ExternalRandomGenerator.Instance.NextScaledRandomInteger(0, objectCount - 1);
                            var obj = objects.ElementAt(objectIdx);
                            var objPayloadFromPile = (byte[])pile.Get(obj.Key);
                            Assert.AreEqual(obj.Value[0], objPayloadFromPile[0]);
                            Assert.AreEqual(obj.Value[obj.Value.Length - 1], objPayloadFromPile[obj.Value.Length - 1]);
                            Assert.AreEqual(obj.Value.Length, objPayloadFromPile.Length);
                        }
                    }

                    // delete
                    var deleteCount = ExternalRandomGenerator.Instance.NextScaledRandomInteger(fromObjCount, toObjCount);
                    for (int i = 0; i < deleteCount; i++)
                    {
                        if (objectCount == 0) break;

                        var objectIdx = ExternalRandomGenerator.Instance.NextScaledRandomInteger(0, objectCount - 1);
                        var obj = objects.ElementAt(objectIdx);

                        Assert.IsTrue(pile.Delete(obj.Key));
                        objects.Remove(obj.Key);
                        objectCount--; objectsSumSize -= obj.Value.Length;
                    }
                }
            }
        }
Exemplo n.º 3
0
        public void VarSizes_Checkboard(bool isParallel, int cnt, int minSz, int maxSz, bool speed)
        {
            using (var pile = new DefaultPile())
            {
              pile.Start();
              pile.AllocMode = speed ? AllocationMode.FavorSpeed : AllocationMode.ReuseSpace;

              var tasks = new List<Task>();
              for(var t=0; t < (isParallel? (System.Environment.ProcessorCount - 1) : 1); t++)
               tasks.Add(
                  Task.Run( () =>
                  {
                     var dict = new Dictionary<PilePointer, dummy>();
                     var priorOdd = PilePointer.Invalid;
                     for(var i=0; i<cnt; i++)
                     {
                       var even = (i&0x01)==0;
                       var data = new dummy{ bin = new byte[12 + NFX.ExternalRandomGenerator.Instance.NextScaledRandomInteger(minSz, maxSz)]};
                       data.bin.WriteBEInt32(0, ExternalRandomGenerator.Instance.NextRandomInteger);
                       data.bin.WriteBEInt32(data.bin.Length-4, ExternalRandomGenerator.Instance.NextRandomInteger);
                       var ptr = pile.Put(data);
                       Assert.IsTrue( ptr.Valid );

                       if (even)
                        dict.Add(ptr, data);
                       else
                       {
                        if (priorOdd.Valid)
                         Assert.IsTrue( pile.Delete(priorOdd) );
                        priorOdd = ptr;
                       }

                       if (i%1000==0)
                        Console.WriteLine("Thread{0} did {1}; allocated {2} bytes, utilized {3} bytes by {4} objects {5} bytes/obj. ",
                                          Thread.CurrentThread.ManagedThreadId,
                                           i,
                                           pile.AllocatedMemoryBytes,
                                           pile.UtilizedBytes,
                                           pile.ObjectCount,
                                           pile.UtilizedBytes / pile.ObjectCount);

                     }
                     Console.WriteLine("Thread {0} Population done, now checking the buffers... {1}",Thread.CurrentThread.ManagedThreadId, DateTime.Now);

                     foreach(var entry in dict)
                       Assert.IsTrue( NFX.IOMiscUtils.MemBufferEquals(entry.Value.bin, (pile.Get(entry.Key) as dummy).bin));

                     Console.WriteLine("Thread {0} DONE. {1}", Thread.CurrentThread.ManagedThreadId, DateTime.Now);
                  })
            );//add

              Task.WaitAll( tasks.ToArray() );
            }
        }
Exemplo n.º 4
0
      public void PileDeleteInLastSegment(int payloadSize)
      {
        using (var pile = new DefaultPile() { SegmentSize = SEG_SIZE})
        {
          pile.Start();

          var pps = new List<PilePointer>();
          while (pile.SegmentCount < 2)
            pps.Add(pile.Put(generatePayload(payloadSize)));

          pile.Delete(pps.Last());
          pps.RemoveAt(pps.Count-1);

          Console.WriteLine("segment count: {0}, segment total count: {1}", pile.SegmentCount, pile.SegmentTotalCount);

          Assert.AreEqual(1, pile.SegmentCount);
          Assert.AreEqual(1, pile.SegmentTotalCount);

        }   
      }
Exemplo n.º 5
0
      public void PileDeleteInMiddleSegment(int payloadSize)
      {
        using (var pile = new DefaultPile() { SegmentSize = SEG_SIZE})
        {
          pile.Start();

          var pps = new List<PilePointer>();
          while (pile.SegmentCount < 4)
            pps.Add(pile.Put(generatePayload(payloadSize)));

          pile.Delete(pps.Last());
          pps.RemoveAt(pps.Count-1);

          var objectsInsegmentCount = pps.Count / 3;

          Console.WriteLine("{0:N0} object in pile, {1:N0} object per segment", pile.ObjectCount, objectsInsegmentCount);

          Assert.AreEqual(3, pile.SegmentCount);
          Assert.AreEqual(3, pile.SegmentTotalCount);

          for (int i = objectsInsegmentCount; i < 2 * objectsInsegmentCount; i++)
          {
            pile.Delete(pps[i]);
          }

          Console.WriteLine("{0:N0} object in pile, {1:N0} segments, {2:N0} segments total", pile.ObjectCount, pile.SegmentCount, pile.SegmentTotalCount);

          Assert.AreEqual(2, pile.SegmentCount);
          Assert.AreEqual(3, pile.SegmentTotalCount);
        }   
      }
Exemplo n.º 6
0
      public void PileSmallObjects(int payloadSize, params int[] freeChunkSizes)
      {
        using (var pile = new DefaultPile() { SegmentSize = SEG_SIZE, AllocMode = AllocationMode.ReuseSpace })
        {
          pile.FreeChunkSizes = freeChunkSizes;

          pile.Start();

          var pps = new List<PilePointer>();
          while (pile.SegmentCount < 2)
            pps.Add(pile.Put(generatePayload(payloadSize)));

          pile.Delete(pps.Last());
          pps.RemoveAt(pps.Count-1);

          Console.WriteLine("just removed the last added payload and segment should be 1 now, real segment count is {0}", pile.SegmentCount);

          var objectsInFirstSegment = pps.Count;

          Console.WriteLine("put {0:N0} objects in first segment, pile.ObjectCount {1:N0}", objectsInFirstSegment, pile.ObjectCount);

          var deletedObjectCount = 0;
          for (int i = 0; i < pps.Count; i+=2, deletedObjectCount++)
            Assert.IsTrue(pile.Delete(pps[i]));

          Console.WriteLine("deleted {0:N0} objects, pile.ObjectCount {1:N0}", deletedObjectCount, pile.ObjectCount);

          var crawlStatus = pile.Crawl(false);
          Console.WriteLine("crawl: {0}", crawlStatus);

          var pps1 = new List<PilePointer>();
          var c = 0;
          while (pile.SegmentCount < 3)
          {
            pps1.Add(pile.Put(generatePayload(payloadSize)));
            if (c%20000 ==0 )pile.Crawl(true); //we do crawl because otherwise the 25000 free index xlots get exhausted AND
            c++;                               //this unit tests does not run long enough to cause Crawl within allocator (5+ seconds)
          }                                    //so we induce Crawl by hand to rebiild indexes

          pile.Delete(pps1.Last());
          pps1.RemoveAt(pps1.Count-1);

          Console.WriteLine("again just removed the last added payload and segment should be 2 now, real segment count is {0}", pile.SegmentCount);

          var objectsInSecondSegment = pps1.Count;

          Console.WriteLine("put {0:N0} objects in second segment, pile.ObjectCount {1:N0}", objectsInSecondSegment, pile.ObjectCount);

          Assert.AreEqual(objectsInFirstSegment * 2, pile.ObjectCount, "Object count in pile with two full segments must be equal to double object count in full first segment!");
        }
      }
Exemplo n.º 7
0
        public static void Put_RandomDelete_ByteArray(int cnt, int durationSec, bool speed, int payloadSizeMin, int payloadSizeMax, int deleteFreq, bool isParallel)
        {
            using (var pile = new DefaultPile())
              {
            pile.AllocMode = speed ? AllocationMode.FavorSpeed : AllocationMode.ReuseSpace;
            pile.Start();
            var startTime = DateTime.UtcNow;
            var tasks = new List<Task>();
            for (var t = 0; t < (isParallel ? (System.Environment.ProcessorCount - 1) : 1); t++)
              tasks.Add(Task.Factory.StartNew(() =>
            {
              var wlc = 0;
              while (true)
              {
                if ((DateTime.UtcNow - startTime).TotalSeconds >= durationSec) return;

                var dict = new Dictionary<int, CheckByteArray>();

                Console.WriteLine("Starting a batch of {0}".Args(cnt));
                for (int i = 0; i < cnt; i++)
                {
                  var payloadSize = NFX.ExternalRandomGenerator.Instance.NextScaledRandomInteger(payloadSizeMin, payloadSizeMax);
                  var val = new byte[payloadSize];
                  val[0] = (byte)NFX.ExternalRandomGenerator.Instance.NextRandomInteger;
                  val[payloadSize - 1] = (byte)NFX.ExternalRandomGenerator.Instance.NextRandomInteger;

                  var ptr = pile.Put(val);

                  var element = new CheckByteArray(ptr, payloadSize - 1, val[0], val[payloadSize - 1]);
                  dict.Add(i, element);

                  if (dict.Count > 0 && i % deleteFreq == 0)
                  {
                    while (true)
                    {
                      var idx = i - NFX.ExternalRandomGenerator.Instance.NextScaledRandomInteger(0, i);

                      CheckByteArray stored;
                      if (dict.TryGetValue(idx, out stored))
                      {
                        ptr = stored.Ptr;
                        pile.Delete(ptr);
                        dict.Remove(idx);
                        break;
                      }
                    }
                  }

                  if (dict.Count > 16 && NFX.ExternalRandomGenerator.Instance.NextScaledRandomInteger(0, 100) > 98)
                  {
                    var toRead = NFX.ExternalRandomGenerator.Instance.NextScaledRandomInteger(8, 64);
                    wlc++;
                    if (wlc % 125 == 0)
                      Console.WriteLine("Thread {0} is reading {1} elements, total {2}"
                        .Args(Thread.CurrentThread.ManagedThreadId, toRead, dict.Count));
                    for (var k = 0; k < toRead; k++)
                    {
                      var kvp = dict.Skip(NFX.ExternalRandomGenerator.Instance.NextScaledRandomInteger(0, dict.Count - 1)).First();
                      var buf = pile.Get(kvp.Value.Ptr) as byte[];
                      Assert.AreEqual(kvp.Value.FirstByte, buf[0]);
                      Assert.AreEqual(kvp.Value.LastByte, buf[kvp.Value.IdxLast]);
                    }
                  }
                }

                Console.WriteLine("Thread {0} is doing final read of {1} elements".Args(Thread.CurrentThread.ManagedThreadId, dict.Count));
                foreach (var kvp in dict)
                {
                  var buf = pile.Get(kvp.Value.Ptr) as byte[];
                  Assert.AreEqual(kvp.Value.FirstByte, buf[0]);
                  Assert.AreEqual(kvp.Value.LastByte, buf[kvp.Value.IdxLast]);
                }
              }
            }, TaskCreationOptions.LongRunning));
            Task.WaitAll(tasks.ToArray());
              }
        }
Exemplo n.º 8
0
        public static void NoGrowth_TRow(bool speed, int durationSec, int countMin, int countMax)
        {
            using (var pile = new DefaultPile())
              {
            pile.AllocMode = speed ? AllocationMode.FavorSpeed : AllocationMode.ReuseSpace;
            pile.Start();
            var startTime = DateTime.UtcNow;
            var tasks = new List<Task>();
            for (var t = 0; t < (System.Environment.ProcessorCount - 1); t++)
              tasks.Add(Task.Factory.StartNew(() =>
            {
              var list = new List<CheckTRow>();
              bool put = true;
              while (true)
              {
                if ((DateTime.UtcNow - startTime).TotalSeconds >= durationSec) return;

                if (put)
                {
                  var cnt = NFX.ExternalRandomGenerator.Instance.NextScaledRandomInteger(countMin, countMax);
                  for (int j = 0; j < cnt; j++)
                  {
                    var val = PersonRow.MakeFake(new GDID());

                    var ptr = pile.Put(val);

                    var element = new CheckTRow(ptr, val.ID, val.Address1);
                    list.Add(element);
                  }
                  Console.WriteLine("Thread {0} put {1} objects".Args(Thread.CurrentThread.ManagedThreadId, list.Count));
                  put = false;
                }
                else
                {
                  Console.WriteLine("Thread {0} deleted {1} objects".Args(Thread.CurrentThread.ManagedThreadId, list.Count));
                  for (var j = 0; j < list.Count; j++)
                  {
                    var element = list[j];
                    var buf = pile.Get(element.Ptr) as PersonRow;
                    Assert.AreEqual(element.Id, buf.ID);
                    Assert.AreEqual(element.Address, buf.Address1);
                    pile.Delete(element.Ptr);
                  }
                  list.Clear();
                  put = true;
                }
              }
            }, TaskCreationOptions.LongRunning));
            Task.WaitAll(tasks.ToArray());
              }
              Console.WriteLine("Test finished.");
        }
Exemplo n.º 9
0
        public static void DeleteSeveral_TRow(bool speed, int durationSec, int putMin, int putMax, int delFactor, bool isParallel)
        {
            using (var pile = new DefaultPile())
              {
            pile.AllocMode = speed ? AllocationMode.FavorSpeed : AllocationMode.ReuseSpace;
            pile.Start();
            var startTime = DateTime.UtcNow;
            var tasks = new List<Task>();
            for (var t = 0; t < (isParallel ? (System.Environment.ProcessorCount - 1) : 1); t++)
              tasks.Add(Task.Factory.StartNew(() =>
            {
              var list = new List<CheckTRow>();
              var wlc = 0;
              while (true)
              {
                if ((DateTime.UtcNow - startTime).TotalSeconds >= durationSec) break;

                var putCount = NFX.ExternalRandomGenerator.Instance.NextScaledRandomInteger(putMin, putMax);
                for (int i = 0; i < putCount; i++)
                {
                  var val = PersonRow.MakeFake(new GDID());
                  var ptr = pile.Put(val);
                  list.Add(new CheckTRow(ptr, val.ID, val.Address1));
                }

                // delete several random elements
                int delCount = putCount / delFactor;
                for (int i = 0; i < delCount; i++)
                {
                  var idx = NFX.ExternalRandomGenerator.Instance.NextScaledRandomInteger(0, list.Count - 1);
                  var ptr = list[idx].Ptr;
                  pile.Delete(ptr);
                  list.RemoveAt(idx);
                }

                // get several random elements
                if (list.Count > 64 && NFX.ExternalRandomGenerator.Instance.NextScaledRandomInteger(0, 100) > 98)
                {
                  var toRead = NFX.ExternalRandomGenerator.Instance.NextScaledRandomInteger(8, 64);
                  wlc++;
                  if (wlc % 125 == 0)
                    Console.WriteLine("Thread {0} is reading {1} elements, total {2}"
                      .Args(Thread.CurrentThread.ManagedThreadId, toRead, list.Count));
                  for (var k = 0; k < toRead; k++)
                  {
                    var element = list[NFX.ExternalRandomGenerator.Instance.NextScaledRandomInteger(0, list.Count - 1)];
                    var buf = pile.Get(element.Ptr) as PersonRow;
                    Assert.IsTrue(element.Id.Equals(buf.ID));
                    Assert.IsTrue(element.Address.Equals(buf.Address1));
                  }
                }
              }

              // total check
              Console.WriteLine("Thread {0} is doing final read of {1} elements, objectCount {2}"
                .Args(Thread.CurrentThread.ManagedThreadId, list.Count, pile.ObjectCount));
              foreach (var element in list)
              {
                var buf = pile.Get(element.Ptr) as PersonRow;
                Assert.IsTrue(element.Id.Equals(buf.ID));
                Assert.IsTrue(element.Address.Equals(buf.Address1));
              }
              return;
            }, TaskCreationOptions.LongRunning));
            Task.WaitAll(tasks.ToArray());
              }
        }
Exemplo n.º 10
0
        public static void Chessboard_ByteArray(bool speed, int durationSec, int payloadSizeMin, int payloadSizeMax, bool isParallel)
        {
            using (var pile = new DefaultPile())
              {
            pile.AllocMode = speed ? AllocationMode.FavorSpeed : AllocationMode.ReuseSpace;
            pile.Start();
            var startTime = DateTime.UtcNow;
            var tasks = new List<Task>();
            for (var t = 0; t < (isParallel ? (System.Environment.ProcessorCount - 1) : 1); t++)
              tasks.Add(Task.Factory.StartNew(() =>
            {
              var list = new List<CheckByteArray>();
              var i = 0;
              var wlc = 0;
              while (true)
              {
                if ((DateTime.UtcNow - startTime).TotalSeconds >= durationSec) break;

                var payloadSize = NFX.ExternalRandomGenerator
                                    .Instance.NextScaledRandomInteger(payloadSizeMin, payloadSizeMax);
                var val = new byte[payloadSize];
                val[0] = (byte)NFX.ExternalRandomGenerator.Instance.NextRandomInteger;
                val[payloadSize - 1] = (byte)NFX.ExternalRandomGenerator.Instance.NextRandomInteger;

                var ptr = pile.Put(val);

                var element = new CheckByteArray(ptr, payloadSize - 1, val[0], val[payloadSize - 1]);
                list.Add(element);

                // delete previous element
                if (list.Count > 1 && i % 2 == 0)
                {
                  ptr = list[list.Count - 2].Ptr;
                  pile.Delete(ptr);
                  list.RemoveAt(list.Count - 2);
                }

                // get several random elements
                if (list.Count > 64 && NFX.ExternalRandomGenerator.Instance.NextScaledRandomInteger(0, 100) > 98)
                {
                  var toRead = NFX.ExternalRandomGenerator.Instance.NextScaledRandomInteger(8, 64);
                  wlc++;
                  if (wlc % 125 == 0)
                    Console.WriteLine("Thread {0} is reading {1} elements, total {2}, Pile objects {3}, Pile segments {4} Pile Bytes {5}"
                      .Args(Thread.CurrentThread.ManagedThreadId, toRead, list.Count, pile.ObjectCount, pile.SegmentCount, pile.AllocatedMemoryBytes));
                  for (var k = 0; k < toRead; k++)
                  {
                    element = list[NFX.ExternalRandomGenerator.Instance.NextScaledRandomInteger(0, list.Count - 1)];
                    var buf = pile.Get(element.Ptr) as byte[];
                    Assert.AreEqual(element.FirstByte, buf[0]);
                    Assert.AreEqual(element.LastByte, buf[element.IdxLast]);
                  }
                }

                if (i == Int32.MaxValue)
                  i = 0;
                else
                  i++;

                if (list.Count == Int32.MaxValue)
                  list = new List<CheckByteArray>();
              }

              // total check
              Console.WriteLine("Thread {0} is doing final read of {1} elements, ObjectCount {2}"
                .Args(Thread.CurrentThread.ManagedThreadId, list.Count, pile.ObjectCount));
              foreach (var element in list)
              {
                var buf = pile.Get(element.Ptr) as byte[];
                Assert.AreEqual(element.FirstByte, buf[0]);
                Assert.AreEqual(element.LastByte, buf[element.IdxLast]);
              }
              return;
            }, TaskCreationOptions.LongRunning));
            Task.WaitAll(tasks.ToArray());
              }
        }