public unsafe void NativeInMemWriteRead()
{
OutputStruct output = default(OutputStruct);
var key1 = new KeyStruct {
kfield1 = 13, kfield2 = 14
};
var value = new ValueStruct {
vfield1 = 23, vfield2 = 24
};
fht.Upsert(&key1, &value, null, 0);
var status = fht.Read(&key1, null, &output, null, 0);
if (status == Status.PENDING)
{
fht.CompletePending(true);
}
else
{
Assert.IsTrue(status == Status.OK);
}
Assert.IsTrue(output.value.vfield1 == value.vfield1);
Assert.IsTrue(output.value.vfield2 == value.vfield2);
}
public unsafe void RecoverAndTest(Guid cprVersion, Guid indexVersion)
{
// Recover
fht.Recover(cprVersion, indexVersion);
// Create array for reading
Empty context;
var inputArray = new Input[numUniqueKeys];
for (int i = 0; i < numUniqueKeys; i++)
{
inputArray[i].adId.adId = i;
inputArray[i].numClicks.numClicks = 0;
}
// Register with thread
fht.StartSession();
// Issue read requests
fixed(Input *input = inputArray)
{
for (var i = 0; i < numUniqueKeys; i++)
{
fht.Read(&((input + i)->adId), null, (Output *)&((input + i)->numClicks), &context, i);
}
}
// Complete all pending requests
fht.CompletePending(true);
// Release
fht.StopSession();
// Set checkpoint directory
Config.CheckpointDirectory = test_path;
// Test outputs
var checkpointInfo = default(HybridLogRecoveryInfo);
checkpointInfo.Recover(cprVersion);
// Compute expected array
long[] expected = new long[numUniqueKeys];
foreach (var guid in checkpointInfo.continueTokens.Keys)
{
var sno = checkpointInfo.continueTokens[guid];
for (long i = 0; i <= sno; i++)
{
var id = i % numUniqueKeys;
expected[id]++;
}
}
int threadCount = 1; // single threaded test
int numCompleted = threadCount - checkpointInfo.continueTokens.Count;
for (int t = 0; t < numCompleted; t++)
{
var sno = numOps;
for (long i = 0; i < sno; i++)
{
var id = i % numUniqueKeys;
expected[id]++;
}
}
// Assert if expected is same as found
for (long i = 0; i < numUniqueKeys; i++)
{
Assert.IsTrue(
expected[i] == inputArray[i].numClicks.numClicks,
"Debug error for AdId {0}: Expected ({1}), Found({2})", inputArray[i].adId.adId, expected[i], inputArray[i].numClicks.numClicks);
}
}
public unsafe void RecoverAndTest(Guid cprVersion, Guid indexVersion)
{
// Recover
fht.Recover(cprVersion, indexVersion);
// Create array for reading
Empty context;
var inputArray = new Input[numUniqueKeys];
for (int i = 0; i < numUniqueKeys; i++)
{
inputArray[i].adId.adId = i;
inputArray[i].numClicks.numClicks = 0;
}
// Register with thread
fht.StartSession();
// Issue read requests
fixed(Input *input = inputArray)
{
for (var i = 0; i < numUniqueKeys; i++)
{
fht.Read(&((input + i)->adId), null, (Output *)&((input + i)->numClicks), &context, i);
}
}
// Complete all pending requests
fht.CompletePending(true);
// Release
fht.StopSession();
// Test outputs
Config.CheckpointDirectory = test_path;
var recoveryInfo = default(HybridLogRecoveryInfo);
recoveryInfo.Recover(cprVersion);
int num_threads = recoveryInfo.numThreads;
DirectoryInfo info = new DirectoryInfo(DirectoryConfiguration.GetHybridLogCheckpointFolder(cprVersion));
List <ExecutionContext> cpr_points = new List <ExecutionContext>();
foreach (var file in info.GetFiles())
{
if (file.Name != "info.dat" && file.Name != "snapshot.dat")
{
using (var reader = new StreamReader(file.FullName))
{
var ctx = new ExecutionContext();
ctx.Load(reader);
cpr_points.Add(ctx);
}
}
}
// Compute expected array
long[] expected = new long[numUniqueKeys];
long[] found = new long[numUniqueKeys];
long sno = cpr_points.First().serialNum;
for (long i = 0; i <= sno; i++)
{
var id = i % numUniqueKeys;
expected[id]++;
}
// Assert if expected is same as found
for (long i = 0; i < numUniqueKeys; i++)
{
Assert.IsTrue(
expected[i] == inputArray[i].numClicks.numClicks,
"Debug error for AdId {0}: Expected ({1}), Found({2})", inputArray[i].adId.adId, expected[i], inputArray[i].numClicks.numClicks);
}
}
public unsafe void Test()
{
// Create array for reading
Empty context;
var inputArray = new Input[numUniqueKeys];
for (int i = 0; i < numUniqueKeys; i++)
{
inputArray[i].adId.adId = i;
inputArray[i].numClicks.numClicks = 0;
}
// Register with thread
fht.StartSession();
// Issue read requests
fixed(Input *input = inputArray)
{
for (var i = 0; i < numUniqueKeys; i++)
{
fht.Read(&((input + i)->adId), null, (Output *)&((input + i)->numClicks), &context, i);
}
}
// Complete all pending requests
fht.CompletePending(true);
// Release
fht.StopSession();
// Compute expected array
long[] expected = new long[numUniqueKeys];
for (long j = 0; j < threadCount; j++)
{
var sno = threadNumOps[j];
for (long i = 0; i < sno; i++)
{
var id = i % numUniqueKeys;
expected[id]++;
}
}
// Assert if expected is same as found
var counts = new Dictionary <long, long>();
var sum = 0L;
for (long i = 0; i < numUniqueKeys; i++)
{
if (expected[i] != inputArray[i].numClicks.numClicks)
{
long diff = inputArray[i].numClicks.numClicks - expected[i];
if (!counts.ContainsKey(diff))
{
counts.Add(diff, 0);
}
counts[diff] = counts[diff] + 1;
sum += diff;
Console.WriteLine("Debug error for AdId {0}: Expected ({1}), Found({2})", inputArray[i].adId.adId, expected[i], inputArray[i].numClicks.numClicks);
}
}
if (sum > 0)
{
foreach (var key in counts.Keys)
{
Console.WriteLine("{0}: {1}", key, counts[key]);
}
Console.WriteLine("Sum : {0:X}, (1 << {1})", sum, Math.Log(sum, 2));
}
Console.WriteLine("Test successful");
}
public unsafe void SimpleRecoveryTest1()
{
log = FasterFactory.CreateLogDevice(TestContext.CurrentContext.TestDirectory + "\\hlog", deleteOnClose: true);
Directory.CreateDirectory(TestContext.CurrentContext.TestDirectory + "\\checkpoints");
fht1 = FasterFactory.Create
<AdId, NumClicks, Input, Output, Empty, SimpleFunctions, ICustomFaster>
(indexSizeBuckets: 128,
logSettings: new LogSettings {
LogDevice = log, MutableFraction = 0.1, MemorySizeBits = 29
},
checkpointSettings: new CheckpointSettings {
CheckpointDir = TestContext.CurrentContext.TestDirectory + "\\checkpoints", CheckPointType = CheckpointType.Snapshot
}
);
fht2 = FasterFactory.Create
<AdId, NumClicks, Input, Output, Empty, SimpleFunctions, ICustomFaster>
(indexSizeBuckets: 128,
logSettings: new LogSettings {
LogDevice = log, MutableFraction = 0.1, MemorySizeBits = 29
},
checkpointSettings: new CheckpointSettings {
CheckpointDir = TestContext.CurrentContext.TestDirectory + "\\checkpoints", CheckPointType = CheckpointType.Snapshot
}
);
int numOps = 5000;
var inputArray = new AdId[numOps];
for (int i = 0; i < numOps; i++)
{
inputArray[i].adId = i;
}
NumClicks value;
Input inputArg;
Output output;
fixed(AdId *input = inputArray)
{
fht1.StartSession();
for (int key = 0; key < numOps; key++)
{
value.numClicks = key;
fht1.Upsert(input + key, &value, null, 0);
}
fht1.TakeFullCheckpoint(out Guid token);
fht1.CompleteCheckpoint(true);
fht1.StopSession();
fht2.Recover(token);
fht2.StartSession();
for (int key = 0; key < numOps; key++)
{
var status = fht2.Read(input + key, &inputArg, &output, null, 0);
if (status == Status.PENDING)
{
fht2.CompletePending(true);
}
else
{
Assert.IsTrue(output.value.numClicks == key);
}
}
fht2.StopSession();
}
log.Close();
fht1.Dispose();
fht2.Dispose();
new DirectoryInfo(TestContext.CurrentContext.TestDirectory + "\\checkpoints").Delete(true);
}
public unsafe void RecoverAndTest(Guid indexToken, Guid hybridLogToken)
{
// Recover
fht.Recover(indexToken, hybridLogToken);
// Create array for reading
Empty context;
var inputArray = new Input[numUniqueKeys];
for (int i = 0; i < numUniqueKeys; i++)
{
inputArray[i].adId.adId = i;
inputArray[i].numClicks.numClicks = 0;
}
// Register with thread
fht.StartSession();
// Issue read requests
fixed(Input *input = inputArray)
{
for (var i = 0; i < numUniqueKeys; i++)
{
fht.Read(&((input + i)->adId), null, (Output *)&((input + i)->numClicks), &context, i);
}
}
// Complete all pending requests
fht.CompletePending(true);
// Release
fht.StopSession();
// Test outputs
var checkpointInfo = default(HybridLogRecoveryInfo);
checkpointInfo.Recover(hybridLogToken);
// Compute expected array
long[] expected = new long[numUniqueKeys];
foreach (var guid in checkpointInfo.continueTokens.Keys)
{
var sno = checkpointInfo.continueTokens[guid];
for (long i = 0; i <= sno; i++)
{
var id = i % numUniqueKeys;
expected[id]++;
}
}
int numCompleted = threadCount - checkpointInfo.continueTokens.Count;
for (int t = 0; t < numCompleted; t++)
{
var sno = numOps;
for (long i = 0; i < sno; i++)
{
var id = i % numUniqueKeys;
expected[id]++;
}
}
// Assert if expected is same as found
var counts = new Dictionary <long, long>();
var sum = 0L;
bool error = false;
for (long i = 0; i < numUniqueKeys; i++)
{
if (expected[i] != inputArray[i].numClicks.numClicks)
{
long diff = inputArray[i].numClicks.numClicks - expected[i];
if (!counts.ContainsKey(diff))
{
counts.Add(diff, 0);
}
counts[diff] = counts[diff] + 1;
sum += diff;
Console.WriteLine("Debug error for AdId {0}: Expected ({1}), Found({2})", inputArray[i].adId.adId, expected[i], inputArray[i].numClicks.numClicks);
error = true;
}
}
if (sum > 0)
{
foreach (var key in counts.Keys)
{
Console.WriteLine("{0}: {1}", key, counts[key]);
}
Console.WriteLine("Sum : {0:X}, (1 << {1})", sum, Math.Log(sum, 2));
}
if (error)
{
Console.WriteLine("Test failed");
}
else
{
Console.WriteLine("Test successful");
}
}