public void LockIsKeptAlive_Failure()
{
var config = new RemoteLockerTesterConfig
{
LockersCount = 2,
LocalRivalOptimization = LocalRivalOptimization.Disabled,
LockTtl = TimeSpan.FromSeconds(5),
LockMetadataTtl = TimeSpan.FromMinutes(1),
KeepLockAliveInterval = TimeSpan.FromSeconds(10),
ChangeLockRowThreshold = 10,
TimestampProviderStochasticType = TimestampProviderStochasticType.None,
CassandraClusterSettings = SingleCassandraNodeSetUpFixture.CreateCassandraClusterSettings(attempts: 1, timeout: TimeSpan.FromSeconds(1)),
};
using (var tester = new RemoteLockerTester(config))
{
var lockId = Guid.NewGuid().ToString();
var lock1 = tester[0].Lock(lockId);
Thread.Sleep(TimeSpan.FromSeconds(3));
Assert.That(tester[1].TryGetLock(lockId, out var lock2), Is.False);
Thread.Sleep(TimeSpan.FromSeconds(4)); // waiting in total: 3 + 4 = 1*1 + 5 + 1 sec
Assert.That(tester[1].TryGetLock(lockId, out lock2), Is.True);
lock2.Dispose();
lock1.Dispose();
}
}
public void LockCancellationToken_IsCancelled_AfterLosingLock()
{
var config = new RemoteLockerTesterConfig
{
LockersCount = 1,
LocalRivalOptimization = LocalRivalOptimization.Disabled,
LockTtl = TimeSpan.FromSeconds(5),
LockMetadataTtl = TimeSpan.FromMinutes(1),
KeepLockAliveInterval = TimeSpan.FromSeconds(3),
ChangeLockRowThreshold = 10,
TimestampProviderStochasticType = TimestampProviderStochasticType.None,
CassandraClusterSettings = SingleCassandraNodeSetUpFixture.CreateCassandraClusterSettings(attempts: 1, timeout: TimeSpan.FromSeconds(1)),
};
using (var tester = new RemoteLockerTester(config))
{
var lockId = Guid.NewGuid().ToString();
using (var lock1 = tester[0].Lock(lockId))
{
Assert.That(lock1.LockAliveToken.IsCancellationRequested, Is.False);
// waiting more than LockTtl * 0.5 with no lock prolongation
Thread.Sleep(config.KeepLockAliveInterval);
Assert.That(lock1.LockAliveToken.IsCancellationRequested, Is.True);
}
}
}
public void Lock()
{
using (var tester = new RemoteLockerTester())
{
var lockId = Guid.NewGuid().ToString();
var lock1 = tester.Lock(lockId);
Assert.That(tester.TryGetLock(lockId, out var lock2), Is.False);
lock1.Dispose();
Assert.That(tester.TryGetLock(lockId, out lock2), Is.True);
lock2.Dispose();
}
}
public void TryLock_DifferentLockIds()
{
using (var tester = new RemoteLockerTester())
{
var lockId1 = Guid.NewGuid().ToString();
var lockId2 = Guid.NewGuid().ToString();
var lockId3 = Guid.NewGuid().ToString();
Assert.That(tester.TryGetLock(lockId1, out var lock1), Is.True);
Assert.That(tester.TryGetLock(lockId2, out var lock2), Is.True);
Assert.That(tester.TryGetLock(lockId3, out var lock3), Is.True);
lock1.Dispose();
lock2.Dispose();
lock3.Dispose();
}
}
public void TryLock_SingleLockId()
{
using (var tester = new RemoteLockerTester())
{
var lockId = Guid.NewGuid().ToString();
Assert.That(tester.TryGetLock(lockId, out var lock1), Is.True);
Assert.That(lock1, Is.Not.Null);
Assert.That(tester.TryGetLock(lockId, out var lock2), Is.False);
Assert.That(lock2, Is.Null);
lock1.Dispose();
Assert.That(tester.TryGetLock(lockId, out lock2), Is.True);
Assert.That(lock2, Is.Not.Null);
lock2.Dispose();
}
}
private static void DoTest(TestConfig cfg)
{
var cassandraOpTimeout = TimeSpan.FromMilliseconds(cfg.TesterConfig.CassandraClusterSettings.Timeout);
var longOpDuration = cfg.TesterConfig.LockTtl.Add(cassandraOpTimeout).Multiply(cfg.TesterConfig.CassandraClusterSettings.Attempts);
var lockIds = Enumerable.Range(0, cfg.Locks).Select(x => Guid.NewGuid().ToString()).ToArray();
var resources = new ConcurrentDictionary <string, Guid>();
var opsCounters = new ConcurrentDictionary <string, int>();
var allowFails = cfg.TesterConfig.CassandraFailProbability.HasValue;
using (var tester = new RemoteLockerTester(cfg.TesterConfig))
{
var stopSignal = new ManualResetEvent(false);
var syncSignal = new ManualResetEvent(true);
Task syncerThread = null;
if (cfg.SyncInterval.HasValue)
{
syncerThread = Task.Factory.StartNew(() =>
{
do
{
syncSignal.Reset();
Thread.Sleep(longOpDuration);
syncSignal.Set();
} while (!stopSignal.WaitOne(cfg.SyncInterval.Value));
});
}
var localTester = tester;
var actions = new Action <MultithreadingTestHelper.RunState> [cfg.TesterConfig.LockersCount];
for (var th = 0; th < actions.Length; th++)
{
var remoteLockCreator = tester[th];
actions[th] = state =>
{
var op = 0;
while (op < cfg.OperationsPerThread)
{
IRemoteLock @lock = null;
var disposed = false;
try
{
if (state.ErrorOccurred)
{
break;
}
var lockIndex = ThreadLocalRandom.Instance.Next(lockIds.Length);
var lockId = lockIds[lockIndex];
@lock = Lock(remoteLockCreator, syncSignal, lockId, state);
if (@lock == null)
{
break;
}
var localOpsCounter = opsCounters.GetOrAdd(lockId, 0);
var resource = Guid.NewGuid();
resources[lockId] = resource;
var opDuration = TimeSpan.FromMilliseconds(ThreadLocalRandom.Instance.Next(1, 47));
if (ThreadLocalRandom.Instance.NextDouble() < cfg.FastRunningOpProbability)
{
opDuration = TimeSpan.Zero;
}
else if (ThreadLocalRandom.Instance.NextDouble() < cfg.LongRunningOpProbability)
{
opDuration = opDuration.Add(longOpDuration);
}
Thread.Sleep(opDuration);
CollectionAssert.AreEqual(new[] { @lock.ThreadId }, localTester.GetThreadsInMainRow(lockId));
Assert.That(localTester.GetThreadsInShadeRow(lockId), Is.Not.Contains(@lock.ThreadId));
var lockMetadata = localTester.GetLockMetadata(lockId);
Assert.That(lockMetadata.ProbableOwnerThreadId, Is.EqualTo(@lock.ThreadId));
Assert.That(resources[lockId], Is.EqualTo(resource));
Assert.That(opsCounters[lockId], Is.EqualTo(localOpsCounter));
if (++localOpsCounter % (cfg.TesterConfig.LockersCount * cfg.OperationsPerThread / 100) == 0)
{
Console.Out.Write(".");
}
opsCounters[lockId] = localOpsCounter;
@lock.Dispose();
disposed = true;
Thread.Sleep(1);
/*
* If Cassandra can fail, then @lock.Dispose() could not delete column from Cassandra.
* But it must be automatically deleted after lockTtl.
* For performance reason we do first check manually, because .After sleeps at least polling interval before first check
*/
if (localTester.GetThreadsInMainRow(lockId).Contains(@lock.ThreadId))
{
Assert.That(() => localTester.GetThreadsInMainRow(lockId), Is.Not
.Contains(@lock.ThreadId)
.After(2 * (int)cfg.TesterConfig.LockTtl.TotalMilliseconds, 100));
}
op++;
}
catch (FailedCassandraClusterException)
{
if (!allowFails)
{
throw;
}
}
finally
{
if (@lock != null && !disposed)
{
@lock.Dispose();
}
}
}
};
}
MultithreadingTestHelper.RunOnSeparateThreads(TimeSpan.FromMinutes(30), actions);
stopSignal.Set();
syncerThread?.Wait();
Assert.That(opsCounters.Sum(x => x.Value), Is.EqualTo(cfg.TesterConfig.LockersCount * cfg.OperationsPerThread));
}
}
