public void Setup()
{
log = new LocalStorageDevice("hybridlog_native.log", deleteOnClose: true);
fht = FasterFactory.Create
<KeyStruct, ValueStruct, InputStruct, OutputStruct, Empty, Functions, ICustomFaster>
(128, log);
fht.StartSession();
}
static unsafe void Main(string[] args)
{
var fht = FasterFactory.Create
<KeyStruct, ValueStruct, InputStruct, OutputStruct, Empty, Functions, ICustomFasterKv>
(128, new LogSettings {
LogDevice = FasterFactory.CreateLogDevice(""), MutableFraction = 0.5
});
fht.StartSession();
OutputStruct output = default(OutputStruct);
var key1 = new KeyStruct {
kfield1 = 13, kfield2 = 14
};
var value = new ValueStruct {
vfield1 = 23, vfield2 = 24
};
// Upsert item into store, and read it back
fht.Upsert(&key1, &value, null, 0);
fht.Read(&key1, null, &output, null, 0);
if ((output.value.vfield1 != value.vfield1) || (output.value.vfield2 != value.vfield2))
{
Console.WriteLine("Error!");
}
else
{
Console.WriteLine("Success!");
}
var key2 = new KeyStruct {
kfield1 = 15, kfield2 = 16
};
var input = new InputStruct {
ifield1 = 25, ifield2 = 26
};
// Two read-modify-write (RMW) operations (sum aggregator)
// Followed by read of result
fht.RMW(&key2, &input, null, 0);
fht.RMW(&key2, &input, null, 0);
fht.Read(&key2, null, &output, null, 0);
if ((output.value.vfield1 != input.ifield1 * 2) || (output.value.vfield2 != input.ifield2 * 2))
{
Console.WriteLine("Error!");
}
else
{
Console.WriteLine("Success!");
}
fht.StopSession();
Console.ReadLine();
}
public SingleThreadedRecoveryTest()
{
// Create FASTER index
var log = FasterFactory.CreateLogDevice(DirectoryConfiguration.GetHybridLogFileName());
fht = FasterFactory.Create
<AdId, NumClicks, Input, Output, Empty, Functions, ICustomFasterKv>
(keySpace, log);
}
public void Setup()
{
var log = FasterFactory.CreateLogDevice(Path.GetTempPath() + "\\hybridlog_native.log");
fht = FasterFactory.Create
<KeyStruct, ValueStruct, InputStruct, OutputStruct, Empty, Functions, ICustomFaster>
(128, log);
fht.StartSession();
}
public SingleThreadedRecoveryTest()
{
// Create FASTER index
var log = FasterFactory.CreateLogDevice("logs\\hlog");
fht = FasterFactory.Create
<AdId, NumClicks, Input, Output, Empty, Functions, ICustomFasterKv>
(keySpace, log, checkpointDir: "logs");
}
static void Main(string[] args)
{
// This sample uses structs, but via the safe API (no pointers)
var fht = FasterFactory.Create
<KeyStruct, ValueStruct, InputStruct, OutputStruct,
Empty, CustomFunctions>
(128, new NullDevice(), new CustomFunctions());
fht.StartSession();
Empty context;
OutputStruct output = new OutputStruct();
var key1 = new KeyStruct {
kfield1 = 13, kfield2 = 14
};
var value = new ValueStruct {
vfield1 = 23, vfield2 = 24
};
fht.Upsert(key1, value, default(Empty), 0);
fht.Read(key1, default(InputStruct), ref output, context, 0);
if ((output.value.vfield1 != value.vfield1) || (output.value.vfield2 != value.vfield2))
{
Console.WriteLine("Error!");
}
else
{
Console.WriteLine("Success!");
}
KeyStruct key2 = new KeyStruct {
kfield1 = 15, kfield2 = 16
};
InputStruct input = new InputStruct {
ifield1 = 25, ifield2 = 26
};
fht.RMW(key2, input, context, 0);
fht.RMW(key2, input, context, 0);
fht.Read(key2, default(InputStruct), ref output, context, 0);
if ((output.value.vfield1 != input.ifield1 * 2) || (output.value.vfield2 != input.ifield2 * 2))
{
Console.WriteLine("Error!");
}
else
{
Console.WriteLine("Success!");
}
fht.StopSession();
Console.ReadLine();
}
public void Setup()
{
log = new LocalStorageDevice(TestContext.CurrentContext.TestDirectory + "\\hybridlog_native.log", deleteOnClose: true);
fht = FasterFactory.Create
<KeyStruct, ValueStruct, InputStruct, OutputStruct, Empty, Functions, ICustomFaster>
(128, new LogSettings {
LogDevice = log
});
fht.StartSession();
}
public static void Setup(TestContext t)
{
var log = FasterFactory.CreateLogDevice(Path.GetTempPath() + "\\hybridlog_object.log");
fht = FasterFactory.Create
<MyKey, MyValue, MyInput, MyOutput, MyContext, MyFunctions>
(indexSizeBuckets: 128, logDevice: log, functions: new MyFunctions(),
LogMutableFraction: 0.1, LogPageSizeBits: 9, LogTotalSizeBytes: 512 * 16
);
fht.StartSession();
}
public static void Setup(TestContext t)
{
log = FasterFactory.CreateLogDevice("hlog", deleteOnClose: true);
objlog = FasterFactory.CreateObjectLogDevice("hlog", deleteOnClose: true);
fht = FasterFactory.Create
<MyKey, MyValue, MyInput, MyOutput, MyContext, MyFunctions>
(indexSizeBuckets: 128, logDevice: log, objectLogDevice: objlog, functions: new MyFunctions(),
LogMutableFraction: 0.1, LogPageSizeBits: 9, LogTotalSizeBytes: 512 * 16
);
fht.StartSession();
}
static void Main(string[] args)
{
var log = FasterFactory.CreateLogDevice(Path.GetTempPath() + "hybridlog.log");
var h = FasterFactory.Create
<Wrap <int>, Wrap <int>, Wrap <int>, Wrap <int>, MyContext, MyFunctions>
(128, new MyFunctions(),
new LogSettings {
LogDevice = log, MemorySizeBits = 29
}
);
h.StartSession();
for (int i = 0; i < 20000; i++)
{
h.RMW(new Wrap <int> {
field = i
}, new Wrap <int> {
field = i
}, default(MyContext), 0);
h.RMW(new Wrap <int> {
field = i
}, new Wrap <int> {
field = i
}, default(MyContext), 0);
if (i % 32 == 0)
{
h.Refresh();
}
}
Wrap <int> g1 = new Wrap <int>();
h.Read(new Wrap <int> {
field = 19999
}, new Wrap <int>(), ref g1, new MyContext(), 0);
h.CompletePending(true);
Wrap <int> g2 = new Wrap <int>();
h.Read(new Wrap <int> {
field = 46
}, new Wrap <int>(), ref g2, new MyContext(), 0);
h.CompletePending(true);
Console.WriteLine("Success!");
Console.ReadLine();
}
public ConcurrentRecoveryTest(int threadCount)
{
this.threadCount = threadCount;
tokens = new List <Guid>();
var log = FasterFactory.CreateLogDevice(DirectoryConfiguration.GetHybridLogFileName());
// Create FASTER index
fht = FasterFactory.Create
<AdId, NumClicks, Input, Output, Empty, Functions, ICustomFasterKv>
(keySpace, log);
numActiveThreads = 0;
inputArrays = new BlockingCollection <Input[]>();
Prepare();
}
public ConcurrentTest(int threadCount)
{
this.threadCount = threadCount;
// Create FASTER index
var log = FasterFactory.CreateLogDevice("logs\\hlog");
fht = FasterFactory.Create
<AdId, NumClicks, Input, Output, Empty, Functions, ICustomFasterKv>
(keySpace, log, checkpointDir: "logs");
numActiveThreads = 0;
inputArrays = new BlockingCollection <Input[]>();
threadNumOps = new long[threadCount];
Prepare();
}
public void Setup()
{
log = FasterFactory.CreateLogDevice(TestContext.CurrentContext.TestDirectory + "\\hlog", deleteOnClose: true);
objlog = FasterFactory.CreateObjectLogDevice(TestContext.CurrentContext.TestDirectory + "\\hlog", deleteOnClose: true);
fht = FasterFactory.Create
<MyKey, MyValue, MyInput, MyOutput, MyContext, MyFunctions>
(indexSizeBuckets: 128, functions: new MyFunctions(),
logSettings: new LogSettings {
LogDevice = log, ObjectLogDevice = objlog, MutableFraction = 0.1, MemorySizeBits = 29
},
checkpointSettings: new CheckpointSettings {
CheckPointType = CheckpointType.FoldOver
}
);
fht.StartSession();
}
public ConcurrentRecoveryTest(int threadCount)
{
this.threadCount = threadCount;
tokens = new List<Guid>();
var log = FasterFactory.CreateLogDevice("logs\\hlog");
// Create FASTER index
fht = FasterFactory.Create
<AdId, NumClicks, Input, Output, Empty, Functions, ICustomFasterKv>
(keySpace, new LogSettings { LogDevice = log }, new CheckpointSettings { CheckpointDir = "logs" });
numActiveThreads = 0;
inputArrays = new BlockingCollection<Input[]>();
Prepare();
}
static void Main(string[] args)
{
var log = FasterFactory.CreateLogDevice(Path.GetTempPath() + "hybridlog");
var objlog = FasterFactory.CreateObjectLogDevice(Path.GetTempPath() + "hybridlog");
var h = FasterFactory.Create
<MyKey, MyValue, MyInput, MyOutput, MyContext, MyFunctions>
(128, new MyFunctions(),
new LogSettings {
LogDevice = log, ObjectLogDevice = objlog, MemorySizeBits = 29
}
);
h.StartSession();
for (int i = 0; i < 20000; i++)
{
h.Upsert(new MyKey {
key = i
}, new MyValue {
value = i
}, default(MyContext), 0);
if (i % 32 == 0)
{
h.Refresh();
}
}
MyOutput g1 = new MyOutput();
h.Read(new MyKey {
key = 23
}, new MyInput(), ref g1, new MyContext(), 0);
h.CompletePending(true);
MyOutput g2 = new MyOutput();
h.Read(new MyKey {
key = 46
}, new MyInput(), ref g2, new MyContext(), 0);
h.CompletePending(true);
Console.WriteLine("Success!");
Console.ReadLine();
}
public void Setup()
{
if (test_path == null)
{
test_path = Path.GetTempPath() + Path.GetRandomFileName();
if (!Directory.Exists(test_path))
{
Directory.CreateDirectory(test_path);
}
}
var log = FasterFactory.CreateLogDevice(test_path + "\\hlog");
fht =
FasterFactory.Create
<AdId, NumClicks, Input, Output, Empty, Functions>
(keySpace, log, checkpointDir: test_path, functions: new Functions());
}
public void Setup()
{
if (test_path == null)
{
test_path = Path.GetTempPath() + Path.GetRandomFileName();
if (!Directory.Exists(test_path))
{
Directory.CreateDirectory(test_path);
}
}
log = FasterFactory.CreateLogDevice(test_path + "\\hlog");
fht =
FasterFactory.Create
<AdId, NumClicks, Input, Output, Empty, Functions, ICustomFaster>
(keySpace, new LogSettings {
LogDevice = log
}, new CheckpointSettings {
CheckpointDir = test_path
});
}
static void Main(string[] args)
{
var log = FasterFactory.CreateLogDevice(Path.GetTempPath() + "hybridlog");
var h = FasterFactory.Create
<CacheKey, CacheValue, CacheInput, CacheOutput, CacheContext, CacheFunctions>
(1L << 20, log, new CacheFunctions());
h.StartSession();
const int max = 10000000;
Stopwatch sw = new Stopwatch();
sw.Start();
for (int i = 0; i < max; i++)
{
if (i % 256 == 0)
{
h.Refresh();
if (i % (1 << 19) == 0)
{
long workingSet = Process.GetCurrentProcess().WorkingSet64;
Console.WriteLine($"{i}: {workingSet / 1048576}M");
}
}
h.Upsert(new CacheKey(i), new CacheValue(i), default(CacheContext), 0);
}
sw.Stop();
Console.WriteLine("Total time to upsert {0} elements: {1:0.000} secs ({2:0.00} inserts/sec)", max, sw.ElapsedMilliseconds / 1000.0, max / (sw.ElapsedMilliseconds / 1000.0));
Console.WriteLine("Issuing uniform random read workload");
var rnd = new Random();
int statusPending = 0;
var o = new CacheOutput();
sw.Restart();
for (int i = 0; i < max; i++)
{
long key = rnd.Next(max);
var status = h.Read(new CacheKey(key), default(CacheInput), ref o, default(CacheContext), 0);
switch (status)
{
case Status.PENDING:
h.CompletePending(true);
statusPending++; break;
case Status.ERROR:
throw new Exception("Error!");
}
if (o.value.value != key)
{
throw new Exception("Read error!");
}
}
sw.Stop();
Console.WriteLine("Total time to read {0} elements: {1:0.000} secs ({2:0.00} reads/sec)", max, sw.ElapsedMilliseconds / 1000.0, max / (sw.ElapsedMilliseconds / 1000.0));
Console.WriteLine($"Reads completed with PENDING: {statusPending}");
Console.WriteLine("Done");
Console.ReadLine();
}
public void LargeObjectTest1()
{
log = FasterFactory.CreateLogDevice(TestContext.CurrentContext.TestDirectory + "\\hlog", deleteOnClose: true);
objlog = FasterFactory.CreateObjectLogDevice(TestContext.CurrentContext.TestDirectory + "\\hlog", deleteOnClose: true);
Directory.CreateDirectory(TestContext.CurrentContext.TestDirectory + "\\checkpoints");
fht1 = FasterFactory.Create
<MyKey, MyLargeValue, MyInput, MyLargeOutput, MyContext, MyLargeFunctions>
(indexSizeBuckets: 128, functions: new MyLargeFunctions(),
logSettings: new LogSettings {
LogDevice = log, ObjectLogDevice = objlog, MutableFraction = 0.1, MemorySizeBits = 29
},
checkpointSettings: new CheckpointSettings {
CheckpointDir = TestContext.CurrentContext.TestDirectory + "\\checkpoints", CheckPointType = CheckpointType.Snapshot
}
);
fht2 = FasterFactory.Create
<MyKey, MyLargeValue, MyInput, MyLargeOutput, MyContext, MyLargeFunctions>
(indexSizeBuckets: 128, functions: new MyLargeFunctions(),
logSettings: new LogSettings {
LogDevice = log, ObjectLogDevice = objlog, MutableFraction = 0.1, MemorySizeBits = 29
},
checkpointSettings: new CheckpointSettings {
CheckpointDir = TestContext.CurrentContext.TestDirectory + "\\checkpoints", CheckPointType = CheckpointType.Snapshot
}
);
int maxSize = 1000;
int numOps = 5000;
//var value = new MyLargeValue(size);
fht1.StartSession();
Random r = new Random(33);
for (int key = 0; key < numOps; key++)
{
var value = new MyLargeValue(1 + r.Next(maxSize));
fht1.Upsert(new MyKey {
key = key
}, value, null, 0);
}
fht1.TakeFullCheckpoint(out Guid token);
fht1.CompleteCheckpoint(true);
fht1.StopSession();
fht1.Dispose();
MyLargeOutput output = new MyLargeOutput();
fht2.Recover(token);
fht2.StartSession();
for (int key = 0; key < numOps; key++)
{
var status = fht2.Read(new MyKey {
key = key
}, new MyInput(), ref output, null, 0);
if (status == Status.PENDING)
{
fht2.CompletePending(true);
}
else
{
for (int i = 0; i < output.value.value.Length; i++)
{
Assert.IsTrue(output.value.value[i] == (byte)(output.value.value.Length + i));
}
}
}
fht2.StopSession();
fht2.Dispose();
log.Close();
objlog.Close();
new DirectoryInfo(TestContext.CurrentContext.TestDirectory + "\\checkpoints").Delete(true);
}
static unsafe void Main(string[] args)
{
// This sample uses the unsafe API of FASTER, and works only for blittable struct types
// Your structs have to implement certain static methods (see the structs for details)
// You also define the interface (ICustomFaster) that will be returned by the factory
// This sample represents the highest performance level for FASTER, at the expense of
// supporting limited types.
var fht = FasterFactory.Create
<KeyStruct, ValueStruct, InputStruct, OutputStruct, Empty, Functions, ICustomFasterKv>
(128, FasterFactory.CreateLogDevice(""), LogMutableFraction: 0.5);
fht.StartSession();
OutputStruct output = default(OutputStruct);
var key1 = new KeyStruct {
kfield1 = 13, kfield2 = 14
};
var value = new ValueStruct {
vfield1 = 23, vfield2 = 24
};
// Upsert item into store, and read it back
fht.Upsert(&key1, &value, null, 0);
fht.Read(&key1, null, &output, null, 0);
if ((output.value.vfield1 != value.vfield1) || (output.value.vfield2 != value.vfield2))
{
Console.WriteLine("Error!");
}
else
{
Console.WriteLine("Success!");
}
var key2 = new KeyStruct {
kfield1 = 15, kfield2 = 16
};
var input = new InputStruct {
ifield1 = 25, ifield2 = 26
};
// Two read-modify-write (RMW) operations (sum aggregator)
// Followed by read of result
fht.RMW(&key2, &input, null, 0);
fht.RMW(&key2, &input, null, 0);
fht.Read(&key2, null, &output, null, 0);
if ((output.value.vfield1 != input.ifield1 * 2) || (output.value.vfield2 != input.ifield2 * 2))
{
Console.WriteLine("Error!");
}
else
{
Console.WriteLine("Success!");
}
fht.StopSession();
Console.ReadLine();
}
public unsafe void SimpleRecoveryTest1()
{
log = FasterFactory.CreateLogDevice(TestContext.CurrentContext.TestDirectory + "\\hlog", deleteOnClose: true);
Directory.CreateDirectory(TestContext.CurrentContext.TestDirectory + "\\checkpoints");
fht1 = FasterFactory.Create
<AdId, NumClicks, Input, Output, Empty, SimpleFunctions, ICustomFaster>
(indexSizeBuckets: 128,
logSettings: new LogSettings {
LogDevice = log, MutableFraction = 0.1, MemorySizeBits = 29
},
checkpointSettings: new CheckpointSettings {
CheckpointDir = TestContext.CurrentContext.TestDirectory + "\\checkpoints", CheckPointType = CheckpointType.Snapshot
}
);
fht2 = FasterFactory.Create
<AdId, NumClicks, Input, Output, Empty, SimpleFunctions, ICustomFaster>
(indexSizeBuckets: 128,
logSettings: new LogSettings {
LogDevice = log, MutableFraction = 0.1, MemorySizeBits = 29
},
checkpointSettings: new CheckpointSettings {
CheckpointDir = TestContext.CurrentContext.TestDirectory + "\\checkpoints", CheckPointType = CheckpointType.Snapshot
}
);
int numOps = 5000;
var inputArray = new AdId[numOps];
for (int i = 0; i < numOps; i++)
{
inputArray[i].adId = i;
}
NumClicks value;
Input inputArg;
Output output;
fixed(AdId *input = inputArray)
{
fht1.StartSession();
for (int key = 0; key < numOps; key++)
{
value.numClicks = key;
fht1.Upsert(input + key, &value, null, 0);
}
fht1.TakeFullCheckpoint(out Guid token);
fht1.CompleteCheckpoint(true);
fht1.StopSession();
fht2.Recover(token);
fht2.StartSession();
for (int key = 0; key < numOps; key++)
{
var status = fht2.Read(input + key, &inputArg, &output, null, 0);
if (status == Status.PENDING)
{
fht2.CompletePending(true);
}
else
{
Assert.IsTrue(output.value.numClicks == key);
}
}
fht2.StopSession();
}
log.Close();
fht1.Dispose();
fht2.Dispose();
new DirectoryInfo(TestContext.CurrentContext.TestDirectory + "\\checkpoints").Delete(true);
}
static void Main(string[] args)
{
var log = FasterFactory.CreateLogDevice(Path.GetTempPath() + "hybridlog.log");
var h = FasterFactory.Create
<
#if BLIT_KEY && GENERIC_BLIT_KEY
CompoundGroupKey <Empty, TimeKey <int> >
#else
MyKey
#endif
,
#if BLIT_VALUE && GENERIC_BLIT_VALUE
WrappedState <long>
#else
MyValue
#endif
,
#if BLIT_INPUT && GENERIC_BLIT_INPUT
WrappedInput <int, long>
#else
MyInput
#endif
,
#if BLIT_INPUT && GENERIC_BLIT_INPUT
WrappedState <long>
#else
MyOutput
#endif
,
#if BLIT_CONTEXT && GENERIC_BLIT_CONTEXT
#else
MyContext
#endif
, MyFunctions>
(128, new MyFunctions(),
new LogSettings {
LogDevice = log, MemorySizeBits = 29
}
);
h.StartSession();
for (int i = 0; i < 20000; i++)
{
var key =
#if BLIT_KEY && GENERIC_BLIT_KEY
new CompoundGroupKey <Empty, TimeKey <int> > { /*= i*/
}
#else
new MyKey {
key = i,
}
#endif
;
var value =
#if BLIT_VALUE && GENERIC_BLIT_VALUE
new WrappedState <long> {
state = i
}
#else
new MyValue {
value = i,
}
#endif
;
h.Upsert(key, value, default(MyContext), 0);
if (i % 32 == 0)
{
h.Refresh();
}
}
var key1 =
#if BLIT_KEY && GENERIC_BLIT_KEY
new CompoundGroupKey <Empty, TimeKey <int> > { /*field = 23*/
}
#else
new MyKey {
key = 23,
}
#endif
;
var input1 =
#if BLIT_INPUT && GENERIC_BLIT_INPUT
new WrappedInput <int, long>()
#else
new MyInput()
#endif
;
#if BLIT_OUTPUT && GENERIC_BLIT_OUTPUT
WrappedState <long> g1 = new WrappedState <long>();
#else
MyOutput g1 = new MyOutput();
#endif
h.Read(key1, input1, ref g1, new MyContext(), 0);
h.CompletePending(true);
var key2 =
#if BLIT_KEY && GENERIC_BLIT_KEY
new CompoundGroupKey <Empty, TimeKey <int> > { /*field = 46*/
}
#else
new MyKey {
key = 46,
}
#endif
;
var input2 =
#if BLIT_INPUT && GENERIC_BLIT_INPUT
new WrappedInput <int, long>()
#else
new MyInput()
#endif
;
#if BLIT_OUTPUT && GENERIC_BLIT_OUTPUT
WrappedState <long> g2 = new WrappedState <long>();
#else
MyOutput g2 = new MyOutput();
#endif
h.Read(key2, input2, ref g2, new MyContext(), 0);
h.CompletePending(true);
Console.WriteLine("Success!");
Console.ReadLine();
}