public unsafe void MultiReadSpanByteKeyTest()
{
using var log = Devices.CreateLogDevice(TestContext.CurrentContext.TestDirectory + "/MultiReadSpanByteKeyTest.log", deleteOnClose: true);
using var fht = new FasterKV <SpanByte, long>(
size: 1L << 20,
new LogSettings { LogDevice = log, MemorySizeBits = 15, PageSizeBits = 12 });
using var session = fht.For(new MultiReadSpanByteKeyTestFunctions()).NewSession <MultiReadSpanByteKeyTestFunctions>();
for (int i = 0; i < 3000; i++)
{
var key = MemoryMarshal.Cast <char, byte>($"{i}".AsSpan());
fixed(byte *_ = key)
session.Upsert(SpanByte.FromFixedSpan(key), i);
}
// Evict all records to disk
fht.Log.FlushAndEvict(true);
for (long key = 0; key < 50; key++)
{
// read each key multiple times
for (int i = 0; i < 10; i++)
{
Assert.AreEqual(key, ReadKey($"{key}"));
}
}
long ReadKey(string keyString)
{
Status status;
var key = MemoryMarshal.Cast <char, byte>(keyString.AsSpan());
fixed(byte *_ = key)
status = session.Read(key: SpanByte.FromFixedSpan(key), out var unused);
// All keys need to be fetched from disk
Assert.AreEqual(Status.PENDING, status);
session.CompletePendingWithOutputs(out var completedOutputs, wait: true);
var count = 0;
var value = 0L;
using (completedOutputs)
{
while (completedOutputs.Next())
{
count++;
Assert.AreEqual(Status.OK, completedOutputs.Current.Status);
value = completedOutputs.Current.Output;
}
}
Assert.AreEqual(1, count);
return(value);
}
}
/// <summary>
/// Puts a new record or replace the existing one in the store.
/// </summary>
/// <param name="key">The key of the record.</param>
/// <param name="value">The record value.</param>
public void Put(ulong key, ref Span <byte> value)
{
var status = Status.OK;
var context = Context.WithUpsert(s => { status = s; });
var fixedSpan = SpanByte.FromFixedSpan(value);
status = _session?.Upsert(ref key, ref fixedSpan, context, 0) ?? Status.ERROR;
if (status == Status.PENDING)
{
_session?.CompletePending(true);
}
if (status == Status.ERROR)
{
throw new InvalidOperationException("Error ocurred when inserting data.");
}
}
public unsafe void SpanByteUnitTest1()
{
Span <byte> payload = stackalloc byte[20];
Span <byte> serialized = stackalloc byte[24];
SpanByte sb = SpanByte.FromFixedSpan(payload);
Assert.IsFalse(sb.Serialized);
Assert.AreEqual(20, sb.Length);
Assert.AreEqual(24, sb.TotalSize);
Assert.AreEqual(20, sb.AsSpan().Length);
Assert.AreEqual(20, sb.AsReadOnlySpan().Length);
fixed(byte *ptr = serialized)
sb.CopyTo(ptr);
ref SpanByte ssb = ref SpanByte.ReinterpretWithoutLength(serialized);
public unsafe void RandomReadCacheTest1()
{
TestUtils.DeleteDirectory(TestUtils.MethodTestDir, wait: true);
var log = Devices.CreateLogDevice(TestUtils.MethodTestDir + "/BasicFasterTests.log", deleteOnClose: true);
var fht = new FasterKV <SpanByte, long>(
size: 1L << 20,
new LogSettings
{
LogDevice = log,
MemorySizeBits = 15,
PageSizeBits = 12,
ReadCacheSettings = new ReadCacheSettings
{
MemorySizeBits = 15,
PageSizeBits = 12,
SecondChanceFraction = 0.1,
}
});
var session = fht.For(new Functions()).NewSession <Functions>();
void Read(int i)
{
var keyString = $"{i}";
var key = MemoryMarshal.Cast <char, byte>(keyString.AsSpan());
fixed(byte *_ = key)
{
var context = new Context();
var sb = SpanByte.FromFixedSpan(key);
long input = i * 2;
long output = 0;
var status = session.Read(ref sb, ref input, ref output, context);
if (status == Status.OK)
{
Assert.AreEqual(input, output);
return;
}
Assert.AreEqual(Status.PENDING, status, $"was not OK or PENDING: {keyString}");
session.CompletePending(wait: true);
}
}
var num = 30000;
// write the values
for (int i = 0; i < num; i++)
{
var keyString = $"{i}";
var key = MemoryMarshal.Cast <char, byte>(keyString.AsSpan());
fixed(byte *_ = key)
{
var sb = SpanByte.FromFixedSpan(key);
Assert.AreEqual(Status.OK, session.Upsert(sb, i * 2));
}
}
// read through the keys in order (works)
for (int i = 0; i < num; i++)
{
Read(i);
}
// pick random keys to read
var r = new Random(2115);
for (int i = 0; i < num; i++)
{
Read(r.Next(num));
}
fht.Dispose();
log.Dispose();
TestUtils.DeleteDirectory(TestUtils.MethodTestDir);
}
/// <summary>
/// This sample shows how our special type called SpanByte can be leverage to use FASTER
/// with variable-length keys and/or values without a separate object log. SpanBytes can
/// easily be created using pinned or fixed memory. A SpanByte is basically a sequence of
/// bytes with a 4-byte integer length header that denotes the size of the payload.
///
/// Underlying SpanByte is the use of "ref struct" as a proxy for pointers to variable-sized
/// memory in C# (we call these VariableLengthStructs).
/// </summary>
static void SpanByteSample()
{
// VarLen types do not need an object log
var log = Devices.CreateLogDevice("hlog.log", deleteOnClose: true);
// Create store
// For custom varlen (not SpanByte), you need to provide IVariableLengthStructSettings and IFasterEqualityComparer
var store = new FasterKV <SpanByte, SpanByte>(
size: 1L << 20,
logSettings: new LogSettings {
LogDevice = log, MemorySizeBits = 15, PageSizeBits = 12
});
// Create session
var s = store.For(new CustomSpanByteFunctions(locking: false)).NewSession <CustomSpanByteFunctions>();
Random r = new Random(100);
// stackalloc implies fixed, so it can be used directly with SpanByte
Span <byte> keyMem = stackalloc byte[1000];
Span <byte> valueMem = stackalloc byte[1000];
for (byte i = 0; i < 200; i++)
{
var keyLen = r.Next(1, 1000);
var key = keyMem.Slice(0, keyLen);
key.Fill(i);
var valLen = r.Next(1, 1000);
var value = valueMem.Slice(0, valLen);
value.Fill((byte)valLen);
// Option 1: Using overload for Span<byte>
s.Upsert(key, value);
}
using (IFasterScanIterator <SpanByte, SpanByte> iterator = store.Log.Scan(store.Log.BeginAddress, store.Log.TailAddress))
{
while (iterator.GetNext(out RecordInfo recordInfo, out SpanByte keyObj, out SpanByte valueObj))
{
Console.WriteLine("Key: " + keyObj.ToByteArray());
}
}
bool success = true;
r = new Random(100);
for (byte i = 0; i < 200; i++)
{
var keyLen = r.Next(1, 1000);
Span <byte> key = keyMem.Slice(0, keyLen);
key.Fill(i);
var valLen = r.Next(1, 1000);
// Option 2: Converting fixed Span<byte> to SpanByte
var status = s.Read(SpanByte.FromFixedSpan(key), out byte[] output, userContext: (byte)valLen);
var expectedValue = valueMem.Slice(0, valLen);
expectedValue.Fill((byte)valLen);
if (status == Status.PENDING)
{
s.CompletePending(true);
}
else
{
if ((status != Status.OK) || (!output.SequenceEqual(expectedValue.ToArray())))
{
success = false;
break;
}
}
}
if (success)
{
Console.WriteLine("Success!");
}
else
{
Console.WriteLine("Error!");
}
s.Dispose();
store.Dispose();
log.Dispose();
}
public static void Run()
{
const bool useRmw = true;
const int baseNumber = 45;
// VarLen types do not need an object log
using var log = Devices.CreateLogDevice("hlog.log", deleteOnClose: true);
// Create store
// For custom varlen (not SpanByte), you need to provide IVariableLengthStructSettings and IFasterEqualityComparer.
// For this test we require record-level locking
using var store = new FasterKV <SpanByte, SpanByte>(
size: 1L << 20,
logSettings: new LogSettings { LogDevice = log, MemorySizeBits = 15, PageSizeBits = 12 }, disableLocking: false);
// Create session for ASCII sums. We require two callback function types to be provided:
// AsciiSumSpanByteFunctions implements RMW callback functions
// AsciiSumVLS implements the callback for computing the length of the result new value, given an old value and an input
using var s = store.For(new AsciiSumSpanByteFunctions()).NewSession <AsciiSumSpanByteFunctions>
(sessionVariableLengthStructSettings: new SessionVariableLengthStructSettings <SpanByte, SpanByte> { valueLength = new AsciiSumVLS() });
// Create key
Span <byte> key = stackalloc byte[10];
key.Fill(23);
var _key = SpanByte.FromFixedSpan(key);
// Create input
Span <byte> input = stackalloc byte[10];
int inputLength = Utils.LongToBytes(baseNumber, input);
var _input = SpanByte.FromFixedSpan(input.Slice(0, inputLength));
if (useRmw)
{
s.RMW(_key, _input); // InitialUpdater to 45 (baseNumber)
s.RMW(_key, _input); // InPlaceUpdater to 90
s.RMW(_key, _input); // CopyUpdater to 135 (due to value size increase)
s.RMW(_key, _input); // InPlaceUpdater to 180
store.Log.Flush(true); // Flush by moving ReadOnly address to Tail (retain records in memory)
s.RMW(_key, _input); // CopyUpdater to 225 (due to immutable source value in read-only region)
store.Log.FlushAndEvict(true); // Flush and evict all records to disk
var _status = s.RMW(_key, _input); // CopyUpdater to 270 (due to immutable source value on disk)
if (_status.IsPending)
{
Console.WriteLine("Error!");
return;
}
s.CompletePending(true); // Wait for IO completion
}
else
{
s.Upsert(_key, _input);
}
// Create output space
Span <byte> output = stackalloc byte[10];
var outputWrapper = new SpanByteAndMemory(SpanByte.FromFixedSpan(output));
var status = s.Read(ref _key, ref outputWrapper);
// Read does not go pending, and the output should fit in the provided space (10 bytes)
// Hence, no Memory will be allocated by FASTER
if (!status.Found || !outputWrapper.IsSpanByte)
{
Console.WriteLine("Error!");
return;
}
// Check result value correctness
if (useRmw)
{
inputLength = Utils.LongToBytes(baseNumber * 6, input);
}
if (!outputWrapper.SpanByte.AsReadOnlySpan().SequenceEqual(input.Slice(0, inputLength)))
{
Console.WriteLine("Error!");
return;
}
Console.WriteLine("AsciiSumSample: Success!");
}