/// <summary>
/// Subscriber to pages as they are getting evicted from main memory
/// </summary>
/// <param name="iter"></param>
public void OnNext(IFasterScanIterator <CacheKey, CacheValue> iter)
{
int size = 0;
while (iter.GetNext(out RecordInfo info, out CacheKey key, out CacheValue value))
{
size += key.GetSize;
if (!info.Tombstone) // ignore deleted values being evicted (they are accounted for by ConcurrentDeleter)
{
size += value.GetSize;
}
}
AddTrackedSize(-size);
// Adjust empty page count to drive towards desired memory utilization
if (store.Log.PageAllocationStabilized())
{
if (TotalSizeBytes > TargetSizeBytes)
{
store.Log.EmptyPageCount++;
}
else
{
store.Log.EmptyPageCount--;
}
}
}
public void OnNext(IFasterScanIterator <MyKey, MyValue> iter)
{
while (iter.GetNext(out _, out MyKey key, out MyValue value))
{
Assert.AreEqual(val, key.key, $"LogObserver.OnNext: key");
Assert.AreEqual(val, value.value, $"LogObserver.OnNext: value");
val++;
}
}
public void OnNext(IFasterScanIterator <Key, Value> v)
{
while (v.GetNext(out RecordInfo info, out Key key, out Value value))
{
observer.OnNext(new Record <Key, Value> {
info = info, key = key, value = value
});
}
}
public void OnNext(IFasterScanIterator <MyKey, MyValue> iter)
{
while (iter.GetNext(out _, out MyKey key, out MyValue value))
{
Assert.IsTrue(key.key == val);
Assert.IsTrue(value.value == val);
val++;
}
}
public static void Run()
{
// VarLen types do not need an object log
var log = Devices.CreateLogDevice("hlog.log", deleteOnClose: true);
// Create store
// For custom varlen (not SpanByte), you need to provide IVariableLengthStructSettings and IFasterEqualityComparer
var store = new FasterKV <SpanByte, SpanByte>(
size: 1L << 20,
logSettings: new LogSettings {
LogDevice = log, MemorySizeBits = 15, PageSizeBits = 12
});
// Create session
var s = store.For(new CustomSpanByteFunctions(locking: false)).NewSession <CustomSpanByteFunctions>();
Random r = new Random(100);
// Here, stackalloc implies fixed, so it can be used directly with SpanByte
// For Span<byte> over heap data (e.g., strings or byte[]), make sure to use
// fixed before the FASTER Read/Upsert/RMW operation.
Span <byte> keyMem = stackalloc byte[1000];
Span <byte> valueMem = stackalloc byte[1000];
byte i;
for (i = 0; i < 200; i++)
{
var keyLen = r.Next(1, 1000);
var key = keyMem.Slice(0, keyLen);
key.Fill(i);
var valLen = r.Next(1, 1000);
var value = valueMem.Slice(0, valLen);
value.Fill((byte)valLen);
// Option 1: Using overload for Span<byte>
s.Upsert(key, value);
}
bool success = true;
i = 0;
using (IFasterScanIterator <SpanByte, SpanByte> iterator = store.Log.Scan(store.Log.BeginAddress, store.Log.TailAddress))
{
while (iterator.GetNext(out RecordInfo recordInfo))
{
ref var key = ref iterator.GetKey();
if (key.ToByteArray()[0] != i++)
{
success = false;
break;
}
}
}
public void OnNext(IFasterScanIterator <KeyStruct, ValueStruct> iter)
{
while (iter.GetNext(out _, out KeyStruct key, out ValueStruct value))
{
Assert.AreEqual(val, key.kfield1);
Assert.AreEqual(val + 1, key.kfield2);
Assert.AreEqual(val, value.vfield1);
Assert.AreEqual(val + 1, value.vfield2);
val++;
}
}
public void OnNext(IFasterScanIterator <KeyStruct, ValueStruct> iter)
{
while (iter.GetNext(out _, out KeyStruct key, out ValueStruct value))
{
Assert.IsTrue(key.kfield1 == val);
Assert.IsTrue(key.kfield2 == val + 1);
Assert.IsTrue(value.vfield1 == val);
Assert.IsTrue(value.vfield2 == val + 1);
val++;
}
}
public void OnNext(IFasterScanIterator <CacheKey, CacheValue> iter)
{
int cnt = 0;
while (iter.GetNext(out RecordInfo info, out CacheKey _, out CacheValue _))
{
if (!info.Tombstone) // ignore deleted records being evicted
{
cnt++;
}
}
Interlocked.Add(ref EvictCount, cnt);
}
public void OnNext(IFasterScanIterator <CacheKey, CacheValue> iter)
{
int size = 0;
while (iter.GetNext(out RecordInfo info, out CacheKey key, out CacheValue value))
{
size += key.GetSize;
if (!info.Tombstone) // ignore deleted records being evicted
{
size += value.GetSize;
}
}
Interlocked.Add(ref storeSize, -size);
}
public FasterKVIterator(FasterKV <Key, Value> fht, Functions functions, long untilAddress)
{
this.fht = fht;
enumerationPhase = 0;
VariableLengthStructSettings <Key, Value> variableLengthStructSettings = null;
if (fht.hlog is VariableLengthBlittableAllocator <Key, Value> varLen)
{
variableLengthStructSettings = new VariableLengthStructSettings <Key, Value>
{
keyLength = varLen.KeyLength,
valueLength = varLen.ValueLength,
};
}
tempKv = new FasterKV <Key, Value>(fht.IndexSize, new LogSettings {
LogDevice = new NullDevice(), ObjectLogDevice = new NullDevice(), MutableFraction = 1
}, comparer: fht.Comparer, variableLengthStructSettings: variableLengthStructSettings);
tempKvSession = tempKv.NewSession <Input, Output, Context, Functions>(functions);
iter1 = fht.Log.Scan(fht.Log.BeginAddress, untilAddress);
}
/// <summary>
/// This sample shows how our special type called SpanByte can be leverage to use FASTER
/// with variable-length keys and/or values without a separate object log. SpanBytes can
/// easily be created using pinned or fixed memory. A SpanByte is basically a sequence of
/// bytes with a 4-byte integer length header that denotes the size of the payload.
///
/// Underlying SpanByte is the use of "ref struct" as a proxy for pointers to variable-sized
/// memory in C# (we call these VariableLengthStructs).
/// </summary>
static void SpanByteSample()
{
// VarLen types do not need an object log
var log = Devices.CreateLogDevice("hlog.log", deleteOnClose: true);
// Create store
// For custom varlen (not SpanByte), you need to provide IVariableLengthStructSettings and IFasterEqualityComparer
var store = new FasterKV <SpanByte, SpanByte>(
size: 1L << 20,
logSettings: new LogSettings {
LogDevice = log, MemorySizeBits = 15, PageSizeBits = 12
});
// Create session
var s = store.For(new CustomSpanByteFunctions(locking: false)).NewSession <CustomSpanByteFunctions>();
Random r = new Random(100);
// stackalloc implies fixed, so it can be used directly with SpanByte
Span <byte> keyMem = stackalloc byte[1000];
Span <byte> valueMem = stackalloc byte[1000];
for (byte i = 0; i < 200; i++)
{
var keyLen = r.Next(1, 1000);
var key = keyMem.Slice(0, keyLen);
key.Fill(i);
var valLen = r.Next(1, 1000);
var value = valueMem.Slice(0, valLen);
value.Fill((byte)valLen);
// Option 1: Using overload for Span<byte>
s.Upsert(key, value);
}
using (IFasterScanIterator <SpanByte, SpanByte> iterator = store.Log.Scan(store.Log.BeginAddress, store.Log.TailAddress))
{
while (iterator.GetNext(out RecordInfo recordInfo, out SpanByte keyObj, out SpanByte valueObj))
{
Console.WriteLine("Key: " + keyObj.ToByteArray());
}
}
bool success = true;
r = new Random(100);
for (byte i = 0; i < 200; i++)
{
var keyLen = r.Next(1, 1000);
Span <byte> key = keyMem.Slice(0, keyLen);
key.Fill(i);
var valLen = r.Next(1, 1000);
// Option 2: Converting fixed Span<byte> to SpanByte
var status = s.Read(SpanByte.FromFixedSpan(key), out byte[] output, userContext: (byte)valLen);
var expectedValue = valueMem.Slice(0, valLen);
expectedValue.Fill((byte)valLen);
if (status == Status.PENDING)
{
s.CompletePending(true);
}
else
{
if ((status != Status.OK) || (!output.SequenceEqual(expectedValue.ToArray())))
{
success = false;
break;
}
}
}
if (success)
{
Console.WriteLine("Success!");
}
else
{
Console.WriteLine("Error!");
}
s.Dispose();
store.Dispose();
log.Dispose();
}
