public void LockIsKeptAlive_Failure() { var config = new RemoteLockerTesterConfig { LockersCount = 2, LocalRivalOptimization = LocalRivalOptimization.Disabled, LockTtl = TimeSpan.FromSeconds(5), LockMetadataTtl = TimeSpan.FromMinutes(1), KeepLockAliveInterval = TimeSpan.FromSeconds(10), ChangeLockRowThreshold = 10, TimestampProviderStochasticType = TimestampProviderStochasticType.None, CassandraClusterSettings = SingleCassandraNodeSetUpFixture.CreateCassandraClusterSettings(attempts: 1, timeout: TimeSpan.FromSeconds(1)), }; using (var tester = new RemoteLockerTester(config)) { var lockId = Guid.NewGuid().ToString(); var lock1 = tester[0].Lock(lockId); Thread.Sleep(TimeSpan.FromSeconds(3)); Assert.That(tester[1].TryGetLock(lockId, out var lock2), Is.False); Thread.Sleep(TimeSpan.FromSeconds(4)); // waiting in total: 3 + 4 = 1*1 + 5 + 1 sec Assert.That(tester[1].TryGetLock(lockId, out lock2), Is.True); lock2.Dispose(); lock1.Dispose(); } }
public void LockCancellationToken_IsCancelled_AfterLosingLock() { var config = new RemoteLockerTesterConfig { LockersCount = 1, LocalRivalOptimization = LocalRivalOptimization.Disabled, LockTtl = TimeSpan.FromSeconds(5), LockMetadataTtl = TimeSpan.FromMinutes(1), KeepLockAliveInterval = TimeSpan.FromSeconds(3), ChangeLockRowThreshold = 10, TimestampProviderStochasticType = TimestampProviderStochasticType.None, CassandraClusterSettings = SingleCassandraNodeSetUpFixture.CreateCassandraClusterSettings(attempts: 1, timeout: TimeSpan.FromSeconds(1)), }; using (var tester = new RemoteLockerTester(config)) { var lockId = Guid.NewGuid().ToString(); using (var lock1 = tester[0].Lock(lockId)) { Assert.That(lock1.LockAliveToken.IsCancellationRequested, Is.False); // waiting more than LockTtl * 0.5 with no lock prolongation Thread.Sleep(config.KeepLockAliveInterval); Assert.That(lock1.LockAliveToken.IsCancellationRequested, Is.True); } } }
public void Lock() { using (var tester = new RemoteLockerTester()) { var lockId = Guid.NewGuid().ToString(); var lock1 = tester.Lock(lockId); Assert.That(tester.TryGetLock(lockId, out var lock2), Is.False); lock1.Dispose(); Assert.That(tester.TryGetLock(lockId, out lock2), Is.True); lock2.Dispose(); } }
public void TryLock_DifferentLockIds() { using (var tester = new RemoteLockerTester()) { var lockId1 = Guid.NewGuid().ToString(); var lockId2 = Guid.NewGuid().ToString(); var lockId3 = Guid.NewGuid().ToString(); Assert.That(tester.TryGetLock(lockId1, out var lock1), Is.True); Assert.That(tester.TryGetLock(lockId2, out var lock2), Is.True); Assert.That(tester.TryGetLock(lockId3, out var lock3), Is.True); lock1.Dispose(); lock2.Dispose(); lock3.Dispose(); } }
public void TryLock_SingleLockId() { using (var tester = new RemoteLockerTester()) { var lockId = Guid.NewGuid().ToString(); Assert.That(tester.TryGetLock(lockId, out var lock1), Is.True); Assert.That(lock1, Is.Not.Null); Assert.That(tester.TryGetLock(lockId, out var lock2), Is.False); Assert.That(lock2, Is.Null); lock1.Dispose(); Assert.That(tester.TryGetLock(lockId, out lock2), Is.True); Assert.That(lock2, Is.Not.Null); lock2.Dispose(); } }
private static void DoTest(TestConfig cfg) { var cassandraOpTimeout = TimeSpan.FromMilliseconds(cfg.TesterConfig.CassandraClusterSettings.Timeout); var longOpDuration = cfg.TesterConfig.LockTtl.Add(cassandraOpTimeout).Multiply(cfg.TesterConfig.CassandraClusterSettings.Attempts); var lockIds = Enumerable.Range(0, cfg.Locks).Select(x => Guid.NewGuid().ToString()).ToArray(); var resources = new ConcurrentDictionary <string, Guid>(); var opsCounters = new ConcurrentDictionary <string, int>(); var allowFails = cfg.TesterConfig.CassandraFailProbability.HasValue; using (var tester = new RemoteLockerTester(cfg.TesterConfig)) { var stopSignal = new ManualResetEvent(false); var syncSignal = new ManualResetEvent(true); Task syncerThread = null; if (cfg.SyncInterval.HasValue) { syncerThread = Task.Factory.StartNew(() => { do { syncSignal.Reset(); Thread.Sleep(longOpDuration); syncSignal.Set(); } while (!stopSignal.WaitOne(cfg.SyncInterval.Value)); }); } var localTester = tester; var actions = new Action <MultithreadingTestHelper.RunState> [cfg.TesterConfig.LockersCount]; for (var th = 0; th < actions.Length; th++) { var remoteLockCreator = tester[th]; actions[th] = state => { var op = 0; while (op < cfg.OperationsPerThread) { IRemoteLock @lock = null; var disposed = false; try { if (state.ErrorOccurred) { break; } var lockIndex = ThreadLocalRandom.Instance.Next(lockIds.Length); var lockId = lockIds[lockIndex]; @lock = Lock(remoteLockCreator, syncSignal, lockId, state); if (@lock == null) { break; } var localOpsCounter = opsCounters.GetOrAdd(lockId, 0); var resource = Guid.NewGuid(); resources[lockId] = resource; var opDuration = TimeSpan.FromMilliseconds(ThreadLocalRandom.Instance.Next(1, 47)); if (ThreadLocalRandom.Instance.NextDouble() < cfg.FastRunningOpProbability) { opDuration = TimeSpan.Zero; } else if (ThreadLocalRandom.Instance.NextDouble() < cfg.LongRunningOpProbability) { opDuration = opDuration.Add(longOpDuration); } Thread.Sleep(opDuration); CollectionAssert.AreEqual(new[] { @lock.ThreadId }, localTester.GetThreadsInMainRow(lockId)); Assert.That(localTester.GetThreadsInShadeRow(lockId), Is.Not.Contains(@lock.ThreadId)); var lockMetadata = localTester.GetLockMetadata(lockId); Assert.That(lockMetadata.ProbableOwnerThreadId, Is.EqualTo(@lock.ThreadId)); Assert.That(resources[lockId], Is.EqualTo(resource)); Assert.That(opsCounters[lockId], Is.EqualTo(localOpsCounter)); if (++localOpsCounter % (cfg.TesterConfig.LockersCount * cfg.OperationsPerThread / 100) == 0) { Console.Out.Write("."); } opsCounters[lockId] = localOpsCounter; @lock.Dispose(); disposed = true; Thread.Sleep(1); /* * If Cassandra can fail, then @lock.Dispose() could not delete column from Cassandra. * But it must be automatically deleted after lockTtl. * For performance reason we do first check manually, because .After sleeps at least polling interval before first check */ if (localTester.GetThreadsInMainRow(lockId).Contains(@lock.ThreadId)) { Assert.That(() => localTester.GetThreadsInMainRow(lockId), Is.Not .Contains(@lock.ThreadId) .After(2 * (int)cfg.TesterConfig.LockTtl.TotalMilliseconds, 100)); } op++; } catch (FailedCassandraClusterException) { if (!allowFails) { throw; } } finally { if (@lock != null && !disposed) { @lock.Dispose(); } } } }; } MultithreadingTestHelper.RunOnSeparateThreads(TimeSpan.FromMinutes(30), actions); stopSignal.Set(); syncerThread?.Wait(); Assert.That(opsCounters.Sum(x => x.Value), Is.EqualTo(cfg.TesterConfig.LockersCount * cfg.OperationsPerThread)); } }