private void Downgrade(UUID seriesId)
{
if (_isConductor)
{
_writeSeriesLocks.Remove(seriesId);
}
LogReleaseLock(seriesId);
}
public unsafe void CouldRecoverFromFailuresScenario4via2()
{
if (!ChaosMonkey.Enabled)
{
Assert.Inconclusive("Chaos monkey must be enabled for recovery tests.");
}
var dm = new PersistentMapFixedLength <long, long>("../CouldRecoveFromFailures");
dm.Clear();
// scenario 4 via 2: fail during adding new key when free list > 0
for (int scenario = 41; scenario <= 43; scenario++)
{
Console.WriteLine($"Scenario {scenario}");
dm[42] = 420;
Assert.IsTrue(dm.Remove(42));
Assert.IsTrue(dm.freeCount > 0);
ChaosMonkey.Force = true;
ChaosMonkey.Scenario = scenario;
var fl = dm.freeList;
var fc = dm.freeCount;
Assert.Throws <ChaosMonkeyException>(() => {
dm[43] = 430;
});
// Now we must have recoverFlags set to (1 << 2)
Assert.AreEqual(1 << 2 | 1 << 4, dm.recoveryFlags);
Assert.AreEqual(fl, dm.freeListCopy);
Assert.AreEqual(fc, dm.freeCountCopy);
Assert.AreEqual(Process.GetCurrentProcess().Id, *(int *)dm._buckets._buffer._data, "Lock must be held by current process");
long thisWasNotStored = 0;
Assert.Throws <KeyNotFoundException>(() => {
// Item.Get should recover and undo set
thisWasNotStored = dm[43];
});
Assert.AreEqual(0, thisWasNotStored);
Assert.AreEqual(0, dm.recoveryFlags, "Must recover from all scenarios");
Assert.AreEqual(0, *(int *)dm._buckets._buffer._data, "Lock must be released");
}
}
public unsafe void CouldRecoverFromFailuresScenario7via5()
{
if (!ChaosMonkey.Enabled)
{
Assert.Inconclusive("Chaos monkey must be enabled for recovery tests.");
}
var dm = new PersistentMapFixedLength <long, long>("../CouldRecoverFromFailuresScenario5");
dm.Clear();
dm[1] = 1;
// scenario 3: fail during adding new key when free list = 0
for (int scenario = 71; scenario <= 75; scenario++)
{
Console.WriteLine($"Scenario {scenario}");
ChaosMonkey.Force = true;
ChaosMonkey.Scenario = scenario;
var initCount = dm.count;
Assert.Throws <ChaosMonkeyException>(() => {
dm.Remove(1);
});
// Now we must have recoverFlags set to (1 << 3)
Assert.AreEqual(1 << 5 | 1 << 7, dm.recoveryFlags);
Assert.AreEqual(Process.GetCurrentProcess().Id, *(int *)dm._buckets._buffer._data, "Lock must be held by current process");
// recover
Assert.AreEqual(1, dm[1]);
Assert.AreEqual(initCount, dm.Count);
Assert.AreEqual(0, dm.recoveryFlags, "Must recover from all scenarios");
Assert.AreEqual(0, *(int *)dm._buckets._buffer._data, "Lock must be released");
}
}
public unsafe void CouldRecoverFromFailuresScenario44()
{
if (!ChaosMonkey.Enabled)
{
Assert.Inconclusive("Chaos monkey must be enabled for recovery tests.");
}
var dm = new PersistentMapFixedLength <long, long>("../CouldRecoveFromFailures");
dm.Clear();
// scenario 44: write everything but fail to exit lock
Console.WriteLine($"Scenario {44}");
dm[42] = 420;
Assert.IsTrue(dm.Remove(42));
Assert.IsTrue(dm.freeCount > 0);
ChaosMonkey.Force = true;
ChaosMonkey.Scenario = 44;
var fc = dm.freeCount;
Assert.Throws <ChaosMonkeyException>(() => {
dm[43] = 430;
});
Assert.AreEqual(0, dm.recoveryFlags);
Assert.AreEqual(dm.freeCount, fc - 1);
Assert.AreEqual(Process.GetCurrentProcess().Id, *(int *)dm._buckets._buffer._data, "Lock must be held by current process");
long thisWasStored = 0;
thisWasStored = dm[43];
Assert.AreEqual(430, thisWasStored);
Assert.AreEqual(0, dm.recoveryFlags, "Must recover from all scenarios");
Assert.AreEqual(0, *(int *)dm._buckets._buffer._data, "Lock must be released");
}
private void Downgrade(UUID seriesId)
{
_writeSeriesLocks.Remove(seriesId);
LogReleaseLock(seriesId);
}
