/// <summary>
/// Create a series repository at a specified location.
/// </summary>
/// <param name="path">A directory path where repository is stored. If null or empty, then
/// a default folder is used.</param>
/// <param name="bufferSizeMb">Buffer size in megabytes. Ignored if below default.</param>
public DataRepository(string path = null, uint bufferSizeMb = 100) : base(GetConnectionStringFromPath(path))
{
if (string.IsNullOrWhiteSpace(path))
{
path = Path.Combine(Bootstrap.Bootstrapper.Instance.DataFolder, "Repos", "Default");
}
var seriesPath = Path.Combine(path, "series");
_mapsPath = Path.Combine(path, "maps");
if (!Directory.Exists(_mapsPath))
{
Directory.CreateDirectory(_mapsPath);
}
var writeLocksFileName = Path.Combine(seriesPath, WriteLocksFileName);
_writeSeriesLocks = new PersistentMapFixedLength <UUID, long>(writeLocksFileName, 1000);
var logBufferFileName = Path.Combine(seriesPath, LogBufferFileName);
_appendLog = new AppendLog(logBufferFileName, bufferSizeMb < MinimumBufferSize ? (int)MinimumBufferSize : (int)bufferSizeMb);
_appendLog.OnAppend += OnLogAppend;
_pid = (((long)_counter) << 32) | (long)Process.GetCurrentProcess().Id;
_counter++;
_isConductor = TryBecomeConductor();
}
public unsafe void CouldRecoverFromFailuresScenario4via2()
{
if (!ChaosMonkey.Enabled)
{
Assert.Inconclusive("Chaos monkey must be enabled for recovery tests.");
}
var dm = new PersistentMapFixedLength <long, long>("../CouldRecoveFromFailures");
dm.Clear();
// scenario 4 via 2: fail during adding new key when free list > 0
for (int scenario = 41; scenario <= 43; scenario++)
{
Console.WriteLine($"Scenario {scenario}");
dm[42] = 420;
Assert.IsTrue(dm.Remove(42));
Assert.IsTrue(dm.freeCount > 0);
ChaosMonkey.Force = true;
ChaosMonkey.Scenario = scenario;
var fl = dm.freeList;
var fc = dm.freeCount;
Assert.Throws <ChaosMonkeyException>(() => {
dm[43] = 430;
});
// Now we must have recoverFlags set to (1 << 2)
Assert.AreEqual(1 << 2 | 1 << 4, dm.recoveryFlags);
Assert.AreEqual(fl, dm.freeListCopy);
Assert.AreEqual(fc, dm.freeCountCopy);
Assert.AreEqual(Process.GetCurrentProcess().Id, *(int *)dm._buckets._buffer._data, "Lock must be held by current process");
long thisWasNotStored = 0;
Assert.Throws <KeyNotFoundException>(() => {
// Item.Get should recover and undo set
thisWasNotStored = dm[43];
});
Assert.AreEqual(0, thisWasNotStored);
Assert.AreEqual(0, dm.recoveryFlags, "Must recover from all scenarios");
Assert.AreEqual(0, *(int *)dm._buckets._buffer._data, "Lock must be released");
}
}
public unsafe void CouldRecoverFromFailuresScenario1()
{
if (!ChaosMonkey.Enabled)
{
Assert.Inconclusive("Chaos monkey must be enabled for recovery tests.");
}
var dm = new PersistentMapFixedLength <long, long>("../CouldRecoveFromFailures");
dm.Clear();
// scenario 1: fail during rewriting existing key
for (int scenario = 11; scenario <= 13; scenario++)
{
Console.WriteLine($"Scenario {scenario}");
dm[42] = 420;
// next set to the key 42 will throw
ChaosMonkey.Force = true;
ChaosMonkey.Scenario = scenario;
Assert.Throws <ChaosMonkeyException>(() => {
dm[42] = 430;
});
// Now we must have recoverFlags set to (1 << 1)
Assert.AreEqual(1 << 1, dm.recoveryFlags);
//var snapshot = dm.entries[-1];
//Assert.AreEqual(42, snapshot.key);
//Assert.AreEqual(420, snapshot.value);
Assert.AreEqual(Process.GetCurrentProcess().Id, *(int *)dm._buckets._buffer._data, "Lock must be held by current process");
// next read should recover value
Assert.AreEqual(420, dm[42]);
Assert.AreEqual(0, dm.recoveryFlags);
}
Assert.AreEqual(0, *(int *)dm._buckets._buffer._data, "Lock must be released");
Assert.AreEqual(0, dm.recoveryFlags, "Must recover from all scenarios");
}
public unsafe void CouldRecoverFromFailuresScenario7via5()
{
if (!ChaosMonkey.Enabled)
{
Assert.Inconclusive("Chaos monkey must be enabled for recovery tests.");
}
var dm = new PersistentMapFixedLength <long, long>("../CouldRecoverFromFailuresScenario5");
dm.Clear();
dm[1] = 1;
// scenario 3: fail during adding new key when free list = 0
for (int scenario = 71; scenario <= 75; scenario++)
{
Console.WriteLine($"Scenario {scenario}");
ChaosMonkey.Force = true;
ChaosMonkey.Scenario = scenario;
var initCount = dm.count;
Assert.Throws <ChaosMonkeyException>(() => {
dm.Remove(1);
});
// Now we must have recoverFlags set to (1 << 3)
Assert.AreEqual(1 << 5 | 1 << 7, dm.recoveryFlags);
Assert.AreEqual(Process.GetCurrentProcess().Id, *(int *)dm._buckets._buffer._data, "Lock must be held by current process");
// recover
Assert.AreEqual(1, dm[1]);
Assert.AreEqual(initCount, dm.Count);
Assert.AreEqual(0, dm.recoveryFlags, "Must recover from all scenarios");
Assert.AreEqual(0, *(int *)dm._buckets._buffer._data, "Lock must be released");
}
}
/// <summary>
/// Create a series repository at a specified location.
/// </summary>
/// <param name="path">A directory path where repository is stored. If null or empty, then
/// a default folder is used.</param>
/// <param name="bufferSizeMb">Buffer size in megabytes. Ignored if below default.</param>
public DataRepository(string path = null, uint bufferSizeMb = 100) : base(GetConnectionStringFromPath(path))
{
//if (!Path.IsPathRooted(path)) {
// path = Path.Combine(Bootstrap.Bootstrapper.Instance.DataFolder, "Repos", path);
//}
var seriesPath = Path.Combine(path, "series");
_mapsPath = Path.Combine(path, "maps");
if (!Directory.Exists(_mapsPath))
{
Directory.CreateDirectory(_mapsPath);
}
var writeLocksFileName = Path.Combine(seriesPath, WriteLocksFileName);
_writeSeriesLocks = new PersistentMapFixedLength <UUID, int>(writeLocksFileName, 1000);
var logBufferFileName = Path.Combine(seriesPath, LogBufferFileName);
_appendLog = new AppendLog(logBufferFileName, bufferSizeMb < MinimumBufferSize ? (int)MinimumBufferSize : (int)bufferSizeMb);
_appendLog.OnAppend += OnLogAppend;
_pid = (_counter << 16) | Process.GetCurrentProcess().Id;
_counter++;
}
public unsafe void CouldRecoverFromFailuresScenario44()
{
if (!ChaosMonkey.Enabled)
{
Assert.Inconclusive("Chaos monkey must be enabled for recovery tests.");
}
var dm = new PersistentMapFixedLength <long, long>("../CouldRecoveFromFailures");
dm.Clear();
// scenario 44: write everything but fail to exit lock
Console.WriteLine($"Scenario {44}");
dm[42] = 420;
Assert.IsTrue(dm.Remove(42));
Assert.IsTrue(dm.freeCount > 0);
ChaosMonkey.Force = true;
ChaosMonkey.Scenario = 44;
var fc = dm.freeCount;
Assert.Throws <ChaosMonkeyException>(() => {
dm[43] = 430;
});
Assert.AreEqual(0, dm.recoveryFlags);
Assert.AreEqual(dm.freeCount, fc - 1);
Assert.AreEqual(Process.GetCurrentProcess().Id, *(int *)dm._buckets._buffer._data, "Lock must be held by current process");
long thisWasStored = 0;
thisWasStored = dm[43];
Assert.AreEqual(430, thisWasStored);
Assert.AreEqual(0, dm.recoveryFlags, "Must recover from all scenarios");
Assert.AreEqual(0, *(int *)dm._buckets._buffer._data, "Lock must be released");
}
public void CouldCRUDDirectDict()
{
var count = 1000000;
var dd = new PersistentMapFixedLength <long, long>("../CouldCRUDDirectDict", count);
//var dd = new Dictionary<DateTime, long>();
var sw = new Stopwatch();
dd.Clear();
var histogram = new LongHistogram(TimeSpan.TicksPerMillisecond * 100 * 10000, 3);
for (int rounds = 0; rounds < 20; rounds++)
{
//dd.Clear();
sw.Restart();
var dtInit = DateTime.Today;
for (int i = 0; i < count; i++)
{
var startTick = sw.ElapsedTicks;
//dd[dtInit.AddTicks(i)] = i;
dd[i] = i;
var ticks = sw.ElapsedTicks - startTick;
var nanos = (long)(1000000000.0 * (double)ticks / Stopwatch.Frequency);
if (rounds >= 1)
{
histogram.RecordValue(nanos);
}
}
Assert.AreEqual(count, dd.Count);
sw.Stop();
Console.WriteLine($"Add elapsed msec: {sw.ElapsedMilliseconds}");
}
histogram.OutputPercentileDistribution(
printStream: Console.Out,
percentileTicksPerHalfDistance: 3,
outputValueUnitScalingRatio: OutputScalingFactor.None);
var histogram2 = new LongHistogram(TimeSpan.TicksPerMillisecond * 100 * 10000, 3);
for (int rounds = 0; rounds < 20; rounds++)
{
sw.Restart();
var sum = 0L;
for (int i = 0; i < count; i++)
{
var startTick = sw.ElapsedTicks;
sum += dd[i];
var ticks = sw.ElapsedTicks - startTick;
var nanos = (long)(1000000000.0 * (double)ticks / Stopwatch.Frequency);
if (rounds >= 1)
{
histogram2.RecordValue(nanos);
}
}
sw.Stop();
Console.WriteLine($"Read elapsed msec: {sw.ElapsedMilliseconds} for sum {sum}");
}
histogram2.OutputPercentileDistribution(
printStream: Console.Out,
percentileTicksPerHalfDistance: 3,
outputValueUnitScalingRatio: OutputScalingFactor.None);
}
