Esempio n. 1
0
        /// <summary>
        /// Subscriber to pages as they are getting evicted from main memory
        /// </summary>
        /// <param name="iter"></param>
        public void OnNext(IFasterScanIterator <CacheKey, CacheValue> iter)
        {
            int size = 0;

            while (iter.GetNext(out RecordInfo info, out CacheKey key, out CacheValue value))
            {
                size += key.GetSize;
                if (!info.Tombstone) // ignore deleted values being evicted (they are accounted for by ConcurrentDeleter)
                {
                    size += value.GetSize;
                }
            }
            AddTrackedSize(-size);

            // Adjust empty page count to drive towards desired memory utilization
            if (store.Log.PageAllocationStabilized())
            {
                if (TotalSizeBytes > TargetSizeBytes)
                {
                    store.Log.EmptyPageCount++;
                }
                else
                {
                    store.Log.EmptyPageCount--;
                }
            }
        }
Esempio n. 2
0
 public void OnNext(IFasterScanIterator <MyKey, MyValue> iter)
 {
     while (iter.GetNext(out _, out MyKey key, out MyValue value))
     {
         Assert.AreEqual(val, key.key, $"LogObserver.OnNext: key");
         Assert.AreEqual(val, value.value, $"LogObserver.OnNext: value");
         val++;
     }
 }
Esempio n. 3
0
 public void OnNext(IFasterScanIterator <Key, Value> v)
 {
     while (v.GetNext(out RecordInfo info, out Key key, out Value value))
     {
         observer.OnNext(new Record <Key, Value> {
             info = info, key = key, value = value
         });
     }
 }
Esempio n. 4
0
 public void OnNext(IFasterScanIterator <MyKey, MyValue> iter)
 {
     while (iter.GetNext(out _, out MyKey key, out MyValue value))
     {
         Assert.IsTrue(key.key == val);
         Assert.IsTrue(value.value == val);
         val++;
     }
 }
Esempio n. 5
0
        public static void Run()
        {
            // VarLen types do not need an object log
            var log = Devices.CreateLogDevice("hlog.log", deleteOnClose: true);

            // Create store
            // For custom varlen (not SpanByte), you need to provide IVariableLengthStructSettings and IFasterEqualityComparer
            var store = new FasterKV <SpanByte, SpanByte>(
                size: 1L << 20,
                    logSettings: new LogSettings {
                LogDevice = log, MemorySizeBits = 15, PageSizeBits = 12
                    });

            // Create session
            var s = store.For(new CustomSpanByteFunctions(locking: false)).NewSession <CustomSpanByteFunctions>();

            Random r = new Random(100);

            // Here, stackalloc implies fixed, so it can be used directly with SpanByte
            // For Span<byte> over heap data (e.g., strings or byte[]), make sure to use
            // fixed before the FASTER Read/Upsert/RMW operation.
            Span <byte> keyMem   = stackalloc byte[1000];
            Span <byte> valueMem = stackalloc byte[1000];

            byte i;

            for (i = 0; i < 200; i++)
            {
                var keyLen = r.Next(1, 1000);
                var key    = keyMem.Slice(0, keyLen);
                key.Fill(i);

                var valLen = r.Next(1, 1000);
                var value  = valueMem.Slice(0, valLen);
                value.Fill((byte)valLen);

                // Option 1: Using overload for Span<byte>
                s.Upsert(key, value);
            }

            bool success = true;

            i = 0;
            using (IFasterScanIterator <SpanByte, SpanByte> iterator = store.Log.Scan(store.Log.BeginAddress, store.Log.TailAddress))
            {
                while (iterator.GetNext(out RecordInfo recordInfo))
                {
                    ref var key = ref iterator.GetKey();
                    if (key.ToByteArray()[0] != i++)
                    {
                        success = false;
                        break;
                    }
                }
            }
Esempio n. 6
0
 public void OnNext(IFasterScanIterator <KeyStruct, ValueStruct> iter)
 {
     while (iter.GetNext(out _, out KeyStruct key, out ValueStruct value))
     {
         Assert.AreEqual(val, key.kfield1);
         Assert.AreEqual(val + 1, key.kfield2);
         Assert.AreEqual(val, value.vfield1);
         Assert.AreEqual(val + 1, value.vfield2);
         val++;
     }
 }
Esempio n. 7
0
 public void OnNext(IFasterScanIterator <KeyStruct, ValueStruct> iter)
 {
     while (iter.GetNext(out _, out KeyStruct key, out ValueStruct value))
     {
         Assert.IsTrue(key.kfield1 == val);
         Assert.IsTrue(key.kfield2 == val + 1);
         Assert.IsTrue(value.vfield1 == val);
         Assert.IsTrue(value.vfield2 == val + 1);
         val++;
     }
 }
Esempio n. 8
0
        public void OnNext(IFasterScanIterator <CacheKey, CacheValue> iter)
        {
            int cnt = 0;

            while (iter.GetNext(out RecordInfo info, out CacheKey _, out CacheValue _))
            {
                if (!info.Tombstone) // ignore deleted records being evicted
                {
                    cnt++;
                }
            }
            Interlocked.Add(ref EvictCount, cnt);
        }
Esempio n. 9
0
        public void OnNext(IFasterScanIterator <CacheKey, CacheValue> iter)
        {
            int size = 0;

            while (iter.GetNext(out RecordInfo info, out CacheKey key, out CacheValue value))
            {
                size += key.GetSize;
                if (!info.Tombstone) // ignore deleted records being evicted
                {
                    size += value.GetSize;
                }
            }
            Interlocked.Add(ref storeSize, -size);
        }
Esempio n. 10
0
        public FasterKVIterator(FasterKV <Key, Value> fht, Functions functions, long untilAddress)
        {
            this.fht         = fht;
            enumerationPhase = 0;

            VariableLengthStructSettings <Key, Value> variableLengthStructSettings = null;

            if (fht.hlog is VariableLengthBlittableAllocator <Key, Value> varLen)
            {
                variableLengthStructSettings = new VariableLengthStructSettings <Key, Value>
                {
                    keyLength   = varLen.KeyLength,
                    valueLength = varLen.ValueLength,
                };
            }

            tempKv = new FasterKV <Key, Value>(fht.IndexSize, new LogSettings {
                LogDevice = new NullDevice(), ObjectLogDevice = new NullDevice(), MutableFraction = 1
            }, comparer: fht.Comparer, variableLengthStructSettings: variableLengthStructSettings);
            tempKvSession = tempKv.NewSession <Input, Output, Context, Functions>(functions);
            iter1         = fht.Log.Scan(fht.Log.BeginAddress, untilAddress);
        }
Esempio n. 11
0
        /// <summary>
        /// This sample shows how our special type called SpanByte can be leverage to use FASTER
        /// with variable-length keys and/or values without a separate object log. SpanBytes can
        /// easily be created using pinned or fixed memory. A SpanByte is basically a sequence of
        /// bytes with a 4-byte integer length header that denotes the size of the payload.
        ///
        /// Underlying SpanByte is the use of "ref struct" as a proxy for pointers to variable-sized
        /// memory in C# (we call these VariableLengthStructs).
        /// </summary>
        static void SpanByteSample()
        {
            // VarLen types do not need an object log
            var log = Devices.CreateLogDevice("hlog.log", deleteOnClose: true);

            // Create store
            // For custom varlen (not SpanByte), you need to provide IVariableLengthStructSettings and IFasterEqualityComparer
            var store = new FasterKV <SpanByte, SpanByte>(
                size: 1L << 20,
                    logSettings: new LogSettings {
                LogDevice = log, MemorySizeBits = 15, PageSizeBits = 12
                    });

            // Create session
            var s = store.For(new CustomSpanByteFunctions(locking: false)).NewSession <CustomSpanByteFunctions>();

            Random r = new Random(100);

            // stackalloc implies fixed, so it can be used directly with SpanByte
            Span <byte> keyMem   = stackalloc byte[1000];
            Span <byte> valueMem = stackalloc byte[1000];

            for (byte i = 0; i < 200; i++)
            {
                var keyLen = r.Next(1, 1000);
                var key    = keyMem.Slice(0, keyLen);
                key.Fill(i);

                var valLen = r.Next(1, 1000);
                var value  = valueMem.Slice(0, valLen);
                value.Fill((byte)valLen);

                // Option 1: Using overload for Span<byte>
                s.Upsert(key, value);
            }

            using (IFasterScanIterator <SpanByte, SpanByte> iterator = store.Log.Scan(store.Log.BeginAddress, store.Log.TailAddress))
            {
                while (iterator.GetNext(out RecordInfo recordInfo, out SpanByte keyObj, out SpanByte valueObj))
                {
                    Console.WriteLine("Key: " + keyObj.ToByteArray());
                }
            }
            bool success = true;

            r = new Random(100);
            for (byte i = 0; i < 200; i++)
            {
                var         keyLen = r.Next(1, 1000);
                Span <byte> key    = keyMem.Slice(0, keyLen);
                key.Fill(i);

                var valLen = r.Next(1, 1000);

                // Option 2: Converting fixed Span<byte> to SpanByte
                var status = s.Read(SpanByte.FromFixedSpan(key), out byte[] output, userContext: (byte)valLen);

                var expectedValue = valueMem.Slice(0, valLen);
                expectedValue.Fill((byte)valLen);

                if (status == Status.PENDING)
                {
                    s.CompletePending(true);
                }
                else
                {
                    if ((status != Status.OK) || (!output.SequenceEqual(expectedValue.ToArray())))
                    {
                        success = false;
                        break;
                    }
                }
            }

            if (success)
            {
                Console.WriteLine("Success!");
            }
            else
            {
                Console.WriteLine("Error!");
            }

            s.Dispose();
            store.Dispose();
            log.Dispose();
        }