public void Setup()
{
inputArray = new AdId[numOps];
for (int i = 0; i < numOps; i++)
{
inputArray[i].adId = i;
}
path = TestContext.CurrentContext.TestDirectory + "\\SimpleAsyncTests\\";
log = Devices.CreateLogDevice(path + "hlog.log", deleteOnClose: true);
Directory.CreateDirectory(path);
fht1 = new FasterKV <long, long>
(1L << 10,
logSettings: new LogSettings {
LogDevice = log, MutableFraction = 1, PageSizeBits = 10, MemorySizeBits = 15
},
checkpointSettings: new CheckpointSettings {
CheckpointDir = path
}
);
}
public void Setup()
{
inputArray = new AdId[numOps];
for (int i = 0; i < numOps; i++)
{
inputArray[i].adId = i;
}
path = TestUtils.MethodTestDir + "/";
TestUtils.RecreateDirectory(path);
log = Devices.CreateLogDevice(path + "Async.log", deleteOnClose: true);
fht1 = new FasterKV <long, long>
(1L << 10,
logSettings: new LogSettings {
LogDevice = log, MutableFraction = 1, PageSizeBits = 10, MemorySizeBits = 15
},
checkpointSettings: new CheckpointSettings {
CheckpointDir = path
}
);
}
/// <summary>
/// Class to track and update cache size
/// </summary>
/// <param name="store">FASTER store instance</param>
/// <param name="targetMemoryBytes">Initial target memory size of FASTER in bytes</param>
public CacheSizeTracker(FasterKV <CacheKey, CacheValue> store, long targetMemoryBytes = long.MaxValue)
{
this.store = store;
if (targetMemoryBytes < long.MaxValue)
{
Console.WriteLine("**** Setting initial target memory: {0,11:N2}KB", targetMemoryBytes / 1024.0);
}
this.TargetSizeBytes = targetMemoryBytes;
Console.WriteLine("Index size: {0}", store.IndexSize * 64);
Console.WriteLine("Total store size: {0}", TotalSizeBytes);
// Register subscriber to receive notifications of log evictions from memory
store.Log.SubscribeEvictions(this);
// Include the separate read cache, if enabled
if (store.ReadCache != null)
{
store.ReadCache.SubscribeEvictions(this);
}
}
/// <summary>
/// Creates a new instance of the frame store.
/// </summary>
/// <param name="folder">Folder to persist changes.</param>
/// <param name="capacity">The expected capacity.</param>
public RawFramesStore(string folder, long capacity)
{
_dataFolder = folder;
_logSizeBits = (int)Math.Log(capacity, 2) + 1;
_logDevice = Devices.CreateLogDevice(@$ "{this._dataFolder}\data\Store-hlog.log", preallocateFile: false);
_fasterKvh = new FasterKV <ulong, SpanByte>
(
size: 1L << _logSizeBits,
logSettings:
new LogSettings
{
LogDevice = this._logDevice,
},
checkpointSettings:
new CheckpointSettings
{
CheckpointDir = $"{this._dataFolder}/data/checkpoints",
CheckPointType = CheckpointType.FoldOver
}
);
}
public RecoveryTest(int threadCount)
{
this.threadCount = threadCount;
numActiveThreads = 0;
// Create FASTER index
var log = Devices.CreateLogDevice("logs\\hlog");
fht = new FasterKV
<AdId, NumClicks, Input, Output, Empty, Functions>
(indexSize, new Functions(),
new LogSettings {
LogDevice = log
},
new CheckpointSettings {
CheckpointDir = "logs"
});
inputArrays = new BlockingCollection <Input[]>();
Prepare();
}
/// <summary>
/// Main program entry point
/// </summary>
static void Main()
{
var path = Path.GetTempPath() + "StoreAsyncApi/";
// Since the example runs forever, we set option to auto-delete files on close
// Note that this setting precludes recovery
var log = Devices.CreateLogDevice(path + "hlog.log", deleteOnClose: true);
var objlog = Devices.CreateLogDevice(path + "hlog.obj.log", deleteOnClose: true);
var logSettings = new LogSettings {
LogDevice = log, ObjectLogDevice = objlog
};
var serializerSettings = new SerializerSettings <CacheKey, CacheValue> {
keySerializer = () => new CacheKeySerializer(), valueSerializer = () => new CacheValueSerializer()
};
faster = new FasterKV <CacheKey, CacheValue>(1L << 20, logSettings, serializerSettings: serializerSettings);
const int NumParallelTasks = 1;
ThreadPool.SetMinThreads(2 * Environment.ProcessorCount, 2 * Environment.ProcessorCount);
TaskScheduler.UnobservedTaskException += (object sender, UnobservedTaskExceptionEventArgs e) =>
{
Console.WriteLine($"Unobserved task exception: {e.Exception}");
e.SetObserved();
};
Task[] tasks = new Task[NumParallelTasks];
for (int i = 0; i < NumParallelTasks; i++)
{
int local = i;
tasks[i] = Task.Run(() => AsyncOperator(local));
}
// Threads for reporting, commit
new Thread(new ThreadStart(ReportThread)).Start();
new Thread(new ThreadStart(CommitThread)).Start();
Task.WaitAll(tasks);
}
public void Setup()
{
log = Devices.CreateLogDevice(TestContext.CurrentContext.TestDirectory + "\\RecoverContinueTests.log", deleteOnClose: true);
Directory.CreateDirectory(TestContext.CurrentContext.TestDirectory + "\\checkpoints3");
fht1 = new FasterKV
<AdId, NumClicks, AdInput, Output, Empty, SimpleFunctions>
(128, new SimpleFunctions(),
logSettings: new LogSettings {
LogDevice = log, MutableFraction = 0.1, MemorySizeBits = 29
},
checkpointSettings: new CheckpointSettings {
CheckpointDir = TestContext.CurrentContext.TestDirectory + "\\checkpoints3", CheckPointType = CheckpointType.Snapshot
}
);
fht2 = new FasterKV
<AdId, NumClicks, AdInput, Output, Empty, SimpleFunctions>
(128, new SimpleFunctions(),
logSettings: new LogSettings {
LogDevice = log, MutableFraction = 0.1, MemorySizeBits = 29
},
checkpointSettings: new CheckpointSettings {
CheckpointDir = TestContext.CurrentContext.TestDirectory + "\\checkpoints3", CheckPointType = CheckpointType.Snapshot
}
);
fht3 = new FasterKV
<AdId, NumClicks, AdInput, Output, Empty, SimpleFunctions>
(128, new SimpleFunctions(),
logSettings: new LogSettings {
LogDevice = log, MutableFraction = 0.1, MemorySizeBits = 29
},
checkpointSettings: new CheckpointSettings {
CheckpointDir = TestContext.CurrentContext.TestDirectory + "\\checkpoints3", CheckPointType = CheckpointType.Snapshot
}
);
numOps = 5000;
}
public unsafe void SpanByteTest1()
{
Span <byte> output = stackalloc byte[20];
SpanByte input = default;
TestUtils.DeleteDirectory(TestUtils.MethodTestDir, wait: true);
try
{
using var log = Devices.CreateLogDevice(TestUtils.MethodTestDir + "/hlog1.log", deleteOnClose: true);
using var fht = new FasterKV <SpanByte, SpanByte>
(128, new LogSettings { LogDevice = log, MemorySizeBits = 17, PageSizeBits = 12 });
using var s = fht.NewSession(new SpanByteFunctions <Empty>());
var key1 = MemoryMarshal.Cast <char, byte>("key1".AsSpan());
var value1 = MemoryMarshal.Cast <char, byte>("value1".AsSpan());
var output1 = SpanByteAndMemory.FromFixedSpan(output);
s.Upsert(key1, value1);
s.Read(key1, ref input, ref output1);
Assert.IsTrue(output1.IsSpanByte);
Assert.IsTrue(output1.SpanByte.AsReadOnlySpan().SequenceEqual(value1));
var key2 = MemoryMarshal.Cast <char, byte>("key2".AsSpan());
var value2 = MemoryMarshal.Cast <char, byte>("value2value2value2".AsSpan());
var output2 = SpanByteAndMemory.FromFixedSpan(output);
s.Upsert(key2, value2);
s.Read(key2, ref input, ref output2);
Assert.IsTrue(!output2.IsSpanByte);
Assert.IsTrue(output2.Memory.Memory.Span.Slice(0, output2.Length).SequenceEqual(value2));
}
finally
{
TestUtils.DeleteDirectory(TestUtils.MethodTestDir);
}
}
public unsafe void VariableLengthTest1()
{
FasterKV <Key, VLValue> fht;
IDevice log;
log = Devices.CreateLogDevice(TestContext.CurrentContext.TestDirectory + "/hlog1.log", deleteOnClose: true);
fht = new FasterKV <Key, VLValue>
(128,
new LogSettings {
LogDevice = log, MemorySizeBits = 17, PageSizeBits = 12
},
null, null, null, new VariableLengthStructSettings <Key, VLValue> {
valueLength = new VLValue()
}
);
var s = fht.NewSession(new VLFunctions());
Input input = default;
Random r = new Random(100);
// Single alloc outside the loop, to the max length we'll need.
int *val = stackalloc int[StackAllocMax];
for (int i = 0; i < 5000; i++)
{
var key1 = new Key {
key = i
};
var len = GetVarLen(r);
ref VLValue value = ref *(VLValue *)val;
for (int j = 0; j < len; j++)
{
*(val + j) = len;
}
s.Upsert(ref key1, ref value, Empty.Default, 0);
}
static void FixedLenServer(string[] args)
{
Console.WriteLine("FASTER fixed-length (binary) KV server");
ParserResult <Options> result = Parser.Default.ParseArguments <Options>(args);
if (result.Tag == ParserResultType.NotParsed)
{
return;
}
var opts = result.MapResult(o => o, xs => new Options());
opts.GetSettings(out var logSettings, out var checkpointSettings, out var indexSize);
// We use blittable structs Key and Value to construct a customized server for fixed-length types
var store = new FasterKV <Key, Value>(indexSize, logSettings, checkpointSettings);
if (opts.Recover)
{
store.Recover();
}
// This fixed-length session provider can be used with compatible clients such as FixedLenClient and FASTER.benchmark
// Uses FixedLenSerializer as our in-built serializer for blittable (fixed length) types
var provider = new FasterKVProvider <Key, Value, Input, Output, Functions, FixedLenSerializer <Key, Value, Input, Output> >(store, e => new Functions());
// Create server
var server = new FasterServer(opts.Address, opts.Port);
// Register session provider for WireFormat.DefaultFixedLenKV
// You can register multiple session providers with the same server, with different wire protocol specifications
server.Register(WireFormat.DefaultFixedLenKV, provider);
// Start server
server.Start();
Console.WriteLine("Started server");
Thread.Sleep(Timeout.Infinite);
}
public static void SaveRandom(long count)
{
var random = RNG.XOrShift1024(Seed1024.Entropic);
var valuesA = random.Stream <long>().Take((uint)count).Freeze();
var valuesB = random.Stream <long>().Take((uint)count).Freeze();
var userContext = StoreContext.Default;
var sw = stopwatch();
var log = OpenLog();
var logConfig = LogConfig(log);
var cpConfig = CpConfig();
var kvs = new FasterKV <Key2, Value2, Input <Value2>, Output <Value2>, StoreContext, KvsExampleOps>
(Pow2.T20, new KvsExampleOps(), logConfig, cpConfig);
kvs.StartSession();
for (var i = 0; i < count; i++)
{
var k = Keys.Key((ulong)DateTime.Now.Ticks);
var v = new Value2
{
vfield1 = valuesA[i],
vfield2 = valuesB[i]
};
kvs.Upsert(ref k, ref v, userContext, 0);
}
kvs.TakeFullCheckpoint(out Guid token);
kvs.CompletePending(true);
kvs.Log.Compact(kvs.Log.HeadAddress);
inform($"Performed {count} insert/read operations in {snapshot(sw)}");
inform($"Checkpoint token {token}");
kvs.StopSession();
CloseLog(log);
}
public void Setup()
{
inputArray = new AdId[numOps];
for (int i = 0; i < numOps; i++)
{
inputArray[i].adId = i;
}
log = Devices.CreateLogDevice(TestUtils.MethodTestDir + "/StateMachineTest1.log", deleteOnClose: true);
string checkpointDir = TestUtils.MethodTestDir + "/statemachinetest";
Directory.CreateDirectory(checkpointDir);
fht1 = new FasterKV <AdId, NumClicks>
(128,
logSettings: new LogSettings {
LogDevice = log, MutableFraction = 0.1, PageSizeBits = 10, MemorySizeBits = 13
},
checkpointSettings: new CheckpointSettings {
CheckpointDir = checkpointDir
}
);
}
static void VarLenServer(string[] args)
{
Console.WriteLine("FASTER variable-length KV server");
ParserResult <Options> result = Parser.Default.ParseArguments <Options>(args);
if (result.Tag == ParserResultType.NotParsed)
{
return;
}
var opts = result.MapResult(o => o, xs => new Options());
opts.GetSettings(out var logSettings, out var checkpointSettings, out var indexSize);
// Create a new instance of the FasterKV, customized for variable-length blittable data (represented by SpanByte)
// With SpanByte, keys and values are stored inline in the FASTER log as [ 4 byte length | payload ]
var store = new FasterKV <SpanByte, SpanByte>(indexSize, logSettings, checkpointSettings);
if (opts.Recover)
{
store.Recover();
}
// This variable-length session provider can be used with compatible clients such as VarLenClient
var provider = new SpanByteFasterKVProvider(store);
// Create server
var server = new FasterServer(opts.Address, opts.Port);
// Register provider as backend provider for WireFormat.DefaultFixedLenKV
// You can register multiple providers with the same server, with different wire protocol specifications
server.Register(WireFormat.DefaultVarLenKV, provider);
// Start server
server.Start();
Console.WriteLine("Started server");
Thread.Sleep(Timeout.Infinite);
}
public async ValueTask RecoveryCheck1([Values] CheckpointType checkpointType, [Values] bool isAsync)
{
using var s1 = fht1.NewSession(new SimpleFunctions <long, long>());
for (long key = 0; key < 1000; key++)
{
s1.Upsert(ref key, ref key);
}
var task = fht1.TakeHybridLogCheckpointAsync(checkpointType);
using var fht2 = new FasterKV <long, long>
(1L << 10,
logSettings: new LogSettings { LogDevice = log, MutableFraction = 1, PageSizeBits = 10, MemorySizeBits = 20 },
checkpointSettings: new CheckpointSettings { CheckpointDir = path }
);
if (isAsync)
{
await task;
await fht2.RecoverAsync();
}
else
{
task.GetAwaiter().GetResult();
fht2.Recover();
}
Assert.IsTrue(fht1.Log.HeadAddress == fht2.Log.HeadAddress);
Assert.IsTrue(fht1.Log.ReadOnlyAddress == fht2.Log.ReadOnlyAddress);
Assert.IsTrue(fht1.Log.TailAddress == fht2.Log.TailAddress);
using var s2 = fht2.NewSession(new SimpleFunctions <long, long>());
for (long key = 0; key < 1000; key++)
{
long output = default;
var status = s2.Read(ref key, ref output);
Assert.IsTrue(status == Status.OK && output == key);
}
}
public bool InitAndRecover()
{
var logSize = 1L << 20;
log = Devices.CreateLogDevice(@$ "{this.dataFolder}\data\Store-hlog.log", preallocateFile: false);
objLog = Devices.CreateLogDevice(@$ "{this.dataFolder}\data\Store-hlog-obj.log", preallocateFile: false);
this.db = new FasterKV
<Types.StoreKey, Types.StoreValue, Types.StoreInput, Types.StoreOutput, Types.StoreContext, Types.StoreFunctions>(
logSize,
new Types.StoreFunctions(),
new LogSettings
{
LogDevice = this.log,
ObjectLogDevice = this.objLog,
MutableFraction = 0.3,
PageSizeBits = 15,
MemorySizeBits = 20
},
new CheckpointSettings
{
CheckpointDir = $"{this.dataFolder}/data/checkpoints"
},
new SerializerSettings <Types.StoreKey, Types.StoreValue>
{
keySerializer = () => new Types.StoreKeySerializer(),
valueSerializer = () => new Types.StoreValueSerializer()
}
);
if (Directory.Exists($"{this.dataFolder}/data/checkpoints"))
{
Console.WriteLine("call recover db");
db.Recover();
return(false);
}
return(true);
}
public void Setup()
{
var readCacheSettings = new ReadCacheSettings {
MemorySizeBits = 15, PageSizeBits = 10
};
log = Devices.CreateLogDevice(TestContext.CurrentContext.TestDirectory + "/ObjectReadCacheTests.log", deleteOnClose: true);
objlog = Devices.CreateLogDevice(TestContext.CurrentContext.TestDirectory + "/ObjectReadCacheTests.obj.log", deleteOnClose: true);
fht = new FasterKV <MyKey, MyValue>
(128,
logSettings: new LogSettings {
LogDevice = log, ObjectLogDevice = objlog, MemorySizeBits = 15, PageSizeBits = 10, ReadCacheSettings = readCacheSettings
},
checkpointSettings: new CheckpointSettings {
CheckPointType = CheckpointType.FoldOver
},
serializerSettings: new SerializerSettings <MyKey, MyValue> {
keySerializer = () => new MyKeySerializer(), valueSerializer = () => new MyValueSerializer()
}
);
}
public void RecoveryCheck4([Values] CheckpointType checkpointType)
{
using var s1 = fht1.NewSession(new SimpleFunctions <long, long>());
using var fht2 = new FasterKV <long, long>
(1L << 10,
logSettings: new LogSettings { LogDevice = log, MutableFraction = 1, PageSizeBits = 10, MemorySizeBits = 20 },
checkpointSettings: new CheckpointSettings { CheckpointDir = path }
);
for (int i = 0; i < 5; i++)
{
for (long key = 1000 * i; key < 1000 * i + 1000; key++)
{
s1.Upsert(ref key, ref key);
}
if (i == 0)
{
fht1.TakeIndexCheckpointAsync().GetAwaiter().GetResult();
}
fht1.TakeHybridLogCheckpointAsync(checkpointType).GetAwaiter().GetResult();
fht2.Recover();
Assert.IsTrue(fht1.Log.HeadAddress == fht2.Log.HeadAddress);
Assert.IsTrue(fht1.Log.ReadOnlyAddress == fht2.Log.ReadOnlyAddress);
Assert.IsTrue(fht1.Log.TailAddress == fht2.Log.TailAddress);
using var s2 = fht2.NewSession(new SimpleFunctions <long, long>());
for (long key = 0; key < 1000 * i + 1000; key++)
{
long output = default;
var status = s2.Read(ref key, ref output);
Assert.IsTrue(status == Status.OK && output == key);
}
}
}
public unsafe void VariableLengthTest1()
{
FasterKV <Key, VLValue, Input, int[], Empty, VLFunctions> fht;
IDevice log;
log = Devices.CreateLogDevice(TestContext.CurrentContext.TestDirectory + "\\hlog1.log", deleteOnClose: true);
fht = new FasterKV <Key, VLValue, Input, int[], Empty, VLFunctions>
(128, new VLFunctions(),
new LogSettings {
LogDevice = log, MemorySizeBits = 17, PageSizeBits = 12
},
null, null, null, new VariableLengthStructSettings <Key, VLValue> {
valueLength = new VLValue()
}
);
fht.StartSession();
Input input = default(Input);
Random r = new Random(100);
for (int i = 0; i < 5000; i++)
{
var key1 = new Key {
key = i
};
var len = 2 + r.Next(10);
int * val = stackalloc int[len];
ref VLValue value = ref *(VLValue *)val;
for (int j = 0; j < len; j++)
{
*(val + j) = len;
}
fht.Upsert(ref key1, ref value, Empty.Default, 0);
}
/// <summary>
/// Create server instance; use Start to start the server.
/// </summary>
/// <param name="opts"></param>
/// <param name="functionsGen"></param>
/// <param name="serializer"></param>
/// <param name="keyInputSerializer"></param>
/// <param name="disableLocking"></param>
/// <param name="maxSizeSettings"></param>
public GenericServer(ServerOptions opts, Func <Functions> functionsGen, ParameterSerializer serializer, IKeyInputSerializer <Key, Input> keyInputSerializer,
bool disableLocking, MaxSizeSettings maxSizeSettings = default)
{
this.opts = opts;
opts.GetSettings(out logSettings, out var checkpointSettings, out var indexSize);
if (opts.EnableStorageTier && !Directory.Exists(opts.LogDir))
{
Directory.CreateDirectory(opts.LogDir);
}
if (!Directory.Exists(opts.CheckpointDir))
{
Directory.CreateDirectory(opts.CheckpointDir);
}
store = new FasterKV <Key, Value>(indexSize, logSettings, checkpointSettings, disableLocking: disableLocking);
if (opts.Recover)
{
try
{
store.Recover();
}
catch { }
}
if (!opts.DisablePubSub)
{
kvBroker = new SubscribeKVBroker <Key, Value, Input, IKeyInputSerializer <Key, Input> >(keyInputSerializer, null, opts.PubSubPageSizeBytes(), true);
broker = new SubscribeBroker <Key, Value, IKeySerializer <Key> >(keyInputSerializer, null, opts.PubSubPageSizeBytes(), true);
}
// Create session provider for VarLen
provider = new FasterKVProvider <Key, Value, Input, Output, Functions, ParameterSerializer>(functionsGen, store, serializer, kvBroker, broker, opts.Recover, maxSizeSettings);
server = new FasterServerTcp(opts.Address, opts.Port);
server.Register(WireFormat.DefaultFixedLenKV, provider);
}
public void Setup()
{
if (test_path == null)
{
test_path = TestContext.CurrentContext.TestDirectory + "/" + Path.GetRandomFileName();
if (!Directory.Exists(test_path))
{
Directory.CreateDirectory(test_path);
}
}
log = Devices.CreateLogDevice(test_path + "/FullRecoveryTests.log");
fht = new FasterKV <AdId, NumClicks>
(keySpace,
new LogSettings {
LogDevice = log
},
new CheckpointSettings {
CheckpointDir = test_path, CheckPointType = CheckpointType.Snapshot
}
);
}
public void Setup()
{
if (test_path == null)
{
test_path = Path.GetTempPath() + Path.GetRandomFileName();
if (!Directory.Exists(test_path))
{
Directory.CreateDirectory(test_path);
}
}
log = Devices.CreateLogDevice(test_path + "\\hlog");
fht = new FasterKV <AdId, NumClicks, Input, Output, Empty, Functions>
(keySpace, new Functions(),
new LogSettings {
LogDevice = log
},
new CheckpointSettings {
CheckpointDir = test_path, CheckPointType = CheckpointType.Snapshot
}
);
}
static void Main()
{
// This sample shows the use of FASTER as a concurrent pure in-memory cache
var log = new NullDevice(); // no storage involved
// Define settings for log
var logSettings = new LogSettings
{
LogDevice = log, ObjectLogDevice = log,
MutableFraction = 0.9, // 10% of memory log is "read-only region"
CopyReadsToTail = CopyReadsToTail.FromReadOnly, // reads in read-only region are copied to tail
PageSizeBits = 14, // Each page is sized at 2^14 bytes
MemorySizeBits = 25, // (2^25 / 24) = ~1.39M key-value pairs (log uses 24 bytes per KV pair)
};
// Number of records in memory, assuming class keys and values and x64 platform
// (8-byte key + 8-byte value + 8-byte header = 24 bytes per record)
int numRecords = (int)(Math.Pow(2, logSettings.MemorySizeBits) / 24);
// Set hash table size targeting 1 record per bucket
var numBucketBits = (int)Math.Ceiling(Math.Log2(numRecords));
h = new FasterKV <CacheKey, CacheValue>(1L << numBucketBits, logSettings, comparer: new CacheKey());
sizeTracker = new CacheSizeTracker(h, targetSize);
// Initially populate store
PopulateStore(numRecords);
// Run continuous read/upsert workload
ContinuousRandomWorkload();
h.Dispose();
Console.WriteLine("Press <ENTER> to end");
Console.ReadLine();
}
// This sample represents an unsafe advanced use of FASTER to store and index variable
// length keys and/or values without a separate object log. The basic idea is to use
// "ref struct" as a proxy for pointers to variable-sized memory in C#. These objects are
// placed contiguously in the single hybrid log, leading to efficient packing while
// avoiding the additional I/O (on reads and writes) that a separate object log entails.
// Users provide information on the actual length of the data underlying the types, by
// providing implementations for an IVariableLengthStruct<T> interface. Serializers are not
// required, as these are effectively value-types. One may provide safe APIs on top of
// this raw functionality using, for example, Span<T> and Memory<T>.
static unsafe void Main()
{
// VarLen types do not need an object log
var log = Devices.CreateLogDevice("hlog.log", deleteOnClose: true);
// Create store, you provide VariableLengthStructSettings for VarLen types
var store = new FasterKV <VarLenType, VarLenType>(
size: 1L << 20,
logSettings: new LogSettings {
LogDevice = log, MemorySizeBits = 17, PageSizeBits = 12
},
comparer: new VarLenTypeComparer(),
variableLengthStructSettings: new VariableLengthStructSettings <VarLenType, VarLenType>
{
keyLength = new VarLenLength(), valueLength = new VarLenLength()
}
);
// Create session
var s = store.For(new VarLenFunctions()).NewSession <VarLenFunctions>();
Random r = new Random(100);
for (int i = 0; i < 5000; i++)
{
var keylen = 2 + r.Next(10);
int * keyval = stackalloc int[keylen];
ref VarLenType key1 = ref *(VarLenType *)keyval;
key1.length = keylen;
for (int j = 1; j < keylen; j++)
{
*(keyval + j) = i;
}
var len = 2 + r.Next(10);
int * val = stackalloc int[len];
ref VarLenType value = ref *(VarLenType *)val;
private unsafe void Populate(FasterKV <VLValue, VLValue> fht)
{
using var session = fht.NewSession(new VLFunctions2());
Random rng = new(RandSeed);
// Single alloc outside the loop, to the max length we'll need.
int *keyval = stackalloc int[StackAllocMax];
int *val = stackalloc int[StackAllocMax];
for (int i = 0; i < DeviceTypeRecoveryTests.numOps; i++)
{
// We must be consistent on length across iterations of each key value
var key0 = i % (int)DeviceTypeRecoveryTests.numUniqueKeys;
if (key0 == 0)
{
rng = new (RandSeed);
}
ref VLValue key1 = ref *(VLValue *)keyval;
key1.length = 2;
key1.field1 = key0;
var len = GetVarLen(rng);
ref VLValue value = ref *(VLValue *)val;
private static void PopulateStore(FasterKV <CacheKey, CacheValue> store)
{
// Start session with FASTER
using var s = store.For(new CacheFunctions()).NewSession <CacheFunctions>();
Console.WriteLine("Writing keys from 0 to {0} to FASTER", numKeys);
Stopwatch sw = new Stopwatch();
sw.Start();
for (int i = 0; i < numKeys; i++)
{
if (i % (1 << 19) == 0)
{
long workingSet = Process.GetCurrentProcess().WorkingSet64;
Console.WriteLine($"{i}: {workingSet / 1048576}M");
}
var key = new CacheKey(i);
var value = new CacheValue(i);
s.Upsert(ref key, ref value);
}
sw.Stop();
Console.WriteLine("Total time to upsert {0} elements: {1:0.000} secs ({2:0.00} inserts/sec)", numKeys, sw.ElapsedMilliseconds / 1000.0, numKeys / (sw.ElapsedMilliseconds / 1000.0));
}
private void CheckAllValues(
ref FasterKV <AdId, NumClicks, Input, Output, Empty, SimpleFunctions> fht,
int value)
{
Input inputArg = default(Input);
Output outputArg = default(Output);
for (var key = 0; key < numOps; key++)
{
inputArg.adId.adId = key;
var status = fht.Read(ref inputArg.adId, ref inputArg, ref outputArg, Empty.Default, key);
if (status == Status.PENDING)
{
fht2.CompletePending(true);
}
else
{
Assert.IsTrue(outputArg.value.numClicks == value);
}
}
fht.CompletePending(true);
}
static void Main(string[] args)
{
// This sample uses class key and value types, which are not blittable (i.e., they
// require a pointer to heap objects). Such datatypes include variable length types
// such as strings. You can override the default key equality comparer in two ways;
// (1) Make Key implement IFasterEqualityComparer<Key> interface
// (2) Provide IFasterEqualityComparer<Key> instance as param to constructor
// FASTER stores the actual objects in a separate object log.
// Note that serializers are required for class types, see below.
var log = Devices.CreateLogDevice(Path.GetTempPath() + "hlog.log");
var objlog = Devices.CreateLogDevice(Path.GetTempPath() + "hlog.obj.log");
var h = new FasterKV
<MyKey, MyValue, MyInput, MyOutput, MyContext, MyFunctions>
(1L << 20, new MyFunctions(),
new LogSettings {
LogDevice = log, ObjectLogDevice = objlog, MemorySizeBits = 29
},
null,
new SerializerSettings <MyKey, MyValue> {
keySerializer = () => new MyKeySerializer(), valueSerializer = () => new MyValueSerializer()
}
);
var context = default(MyContext);
// Each thread calls StartSession to register itself with FASTER
h.StartSession();
for (int i = 0; i < 20000; i++)
{
var _key = new MyKey {
key = i
};
var value = new MyValue {
value = i
};
h.Upsert(ref _key, ref value, context, 0);
// Each thread calls Refresh periodically for thread coordination
if (i % 1024 == 0)
{
h.Refresh();
}
}
var key = new MyKey {
key = 23
};
var input = default(MyInput);
MyOutput g1 = new MyOutput();
var status = h.Read(ref key, ref input, ref g1, context, 0);
if (status == Status.OK && g1.value.value == key.key)
{
Console.WriteLine("Success!");
}
else
{
Console.WriteLine("Error!");
}
MyOutput g2 = new MyOutput();
key = new MyKey {
key = 46
};
status = h.Read(ref key, ref input, ref g2, context, 0);
if (status == Status.OK && g2.value.value == key.key)
{
Console.WriteLine("Success!");
}
else
{
Console.WriteLine("Error!");
}
/// Delete key, read to verify deletion
var output = new MyOutput();
h.Delete(ref key, context, 0);
status = h.Read(ref key, ref input, ref output, context, 0);
if (status == Status.NOTFOUND)
{
Console.WriteLine("Success!");
}
else
{
Console.WriteLine("Error!");
}
// Each thread ends session when done
h.StopSession();
// Dispose FASTER instance and log
h.Dispose();
log.Close();
objlog.Close();
Console.WriteLine("Press <ENTER> to end");
Console.ReadLine();
}
private void Write(ClientSession <MyKey, MyValue, MyInput, MyOutput, MyContext, MyFunctions> session, MyContext context, FasterKV <MyKey, MyValue> fht)
{
for (int i = 0; i < iterations; i++)
{
var _key = new MyKey {
key = i, name = i.ToString()
};
var value = new MyValue {
value = i.ToString()
};
session.Upsert(ref _key, ref value, context, 0);
if (i % 100 == 0)
{
fht.TakeFullCheckpoint(out _);
fht.CompleteCheckpointAsync().GetAwaiter().GetResult();
}
}
}
private void Prepare(CheckpointType checkpointType, out string logPath, out string objPath, out IDevice log, out IDevice objlog, out FasterKV <MyKey, MyValue> h, out MyContext context)
{
logPath = Path.Combine(FasterFolderPath, $"FasterRecoverTests.log");
objPath = Path.Combine(FasterFolderPath, $"FasterRecoverTests_HEAP.log");
log = Devices.CreateLogDevice(logPath);
objlog = Devices.CreateLogDevice(objPath);
h = new FasterKV
<MyKey, MyValue>
(1L << 20,
new LogSettings
{
LogDevice = log,
ObjectLogDevice = objlog,
SegmentSizeBits = 12,
MemorySizeBits = 12,
PageSizeBits = 9
},
new CheckpointSettings()
{
CheckpointDir = Path.Combine(FasterFolderPath, "check-points"),
CheckPointType = checkpointType
},
new SerializerSettings <MyKey, MyValue> {
keySerializer = () => new MyKeySerializer(), valueSerializer = () => new MyValueSerializer()
}
);
context = new MyContext();
}
static void Main()
{
// This sample shows the use of FASTER as a concurrent (multi-threaded) cache + key-value store.
// Number of caches and number of threads can be varied.
h = new FasterKV <CacheKey, CacheValue> [kNumTables];
var path = Path.GetTempPath() + "CacheStoreConcurrent/";
for (int ht = 0; ht < kNumTables; ht++)
{
// Create files for storing data
// We set deleteOnClose to true, so logs will auto-delete on completion
var log = Devices.CreateLogDevice(path + "hlog" + ht + ".log", deleteOnClose: true);
var objlog = Devices.CreateLogDevice(path + "hlog" + ht + ".obj.log", deleteOnClose: true);
// Define settings for log
var logSettings = new LogSettings
{
LogDevice = log,
ObjectLogDevice = objlog,
ReadCacheSettings = useReadCache ? new ReadCacheSettings() : null
};
// Define serializers; otherwise FASTER will use the slower DataContract
// Needed only for class keys/values
var serializerSettings = new SerializerSettings <CacheKey, CacheValue>
{
keySerializer = () => new CacheKeySerializer(),
valueSerializer = () => new CacheValueSerializer()
};
h[ht] = new FasterKV
<CacheKey, CacheValue>(
1L << 20, logSettings,
checkpointSettings: new CheckpointSettings {
CheckpointDir = path
},
serializerSettings: serializerSettings,
comparer: new CacheKey(0)
);
PopulateStore(h[ht]);
// ******
// Uncomment below to move entire log to disk and eliminate data from memory as
// well. This will serve workload entirely from disk using read cache if enabled.
// This will *prevent* future updates to the store. The in-mem buffer is no longer
// allocated.
// h[ht].Log.DisposeFromMemory();
}
ContinuousRandomReadWorkload();
for (int ht = 0; ht < kNumTables; ht++)
{
h[ht].Dispose();
}
Console.WriteLine("Press <ENTER> to end");
Console.ReadLine();
}
