public bool TryGetLock([NotNull] string lockId, out IRemoteLock remoteLock)
{
var threadId = Guid.NewGuid().ToString();
void FinalAction(TimeSpan elapsed)
{
if (elapsed < lockOperationWarnThreshold)
{
return;
}
metrics.FreezeEvents.Mark("TryGetLock");
logger.Error("TryGetLock() took {0} ms for lockId: {1}, threadId: {2}", elapsed.TotalMilliseconds, lockId, threadId);
}
using (metrics.TryGetLockOp.NewContext(FinalAction, FormatLockOperationId(lockId, threadId)))
{
remoteLock = TryAcquireLock(lockId, threadId, out _);
return(remoteLock != null);
}
}
private LocalTaskProcessingResult TryProcessTaskExclusively([NotNull] MetricsContext metricsContext)
{
metricsContext = metricsContext.SubContext(nameof(TryProcessTaskExclusively));
using (metricsContext.Timer("Total").NewContext())
{
IRemoteLock taskGroupRemoteLock = null;
if (!string.IsNullOrEmpty(taskMeta.TaskGroupLock))
{
using (metricsContext.Timer("TryGetLock_TaskGroupLock").NewContext())
{
if (!remoteLockCreator.TryGetLock(taskMeta.TaskGroupLock, out taskGroupRemoteLock))
{
taskShardMetricsContext.Meter("DidNotGetTaskGroupLock").Mark();
logger.Debug("Не смогли взять групповую блокировку {RtqTaskGroupLock} на задачу: {RtqTaskId}", new { RtqTaskGroupLock = taskMeta.TaskGroupLock, RtqTaskId = taskIndexRecord.TaskId });
return(LocalTaskProcessingResult.Undefined);
}
}
taskShardMetricsContext.Meter("GotTaskGroupLock").Mark();
}
try
{
var sw = Stopwatch.StartNew();
IRemoteLock remoteLock;
using (metricsContext.Timer("TryGetLock_TaskId").NewContext())
{
if (!remoteLockCreator.TryGetLock(taskIndexRecord.TaskId, out remoteLock))
{
taskShardMetricsContext.Meter("DidNotGetTaskLock").Mark();
logger.Debug("Не смогли взять блокировку на задачу, пропускаем её: {RtqTaskIndexRecord}", new { RtqTaskIndexRecord = taskIndexRecord });
return(LocalTaskProcessingResult.Undefined);
}
}
taskShardMetricsContext.Meter("GotTaskLock").Mark();
LocalTaskProcessingResult result;
using (remoteLock)
result = ProcessTask(metricsContext);
sw.Stop();
var isLongRunningTask = sw.Elapsed > longRunningTaskDurationThreshold;
logger.Log(new LogEvent(
level: isLongRunningTask ? LogLevel.Warn : LogLevel.Debug,
timestamp: DateTimeOffset.Now,
messageTemplate: "Завершили выполнение задачи {RtqTaskId} с результатом {Result}. Отпустили блокировку {LockId}. Время работы с учетом взятия лока: {Elapsed}")
.WithObjectProperties(new
{
RtqTaskId = taskMeta.Id,
Result = result,
LockId = taskIndexRecord.TaskId,
Elapsed = sw.Elapsed,
IsLongRunningTask = isLongRunningTask,
}));
return(result);
}
finally
{
if (taskGroupRemoteLock != null)
{
taskGroupRemoteLock.Dispose();
logger.Debug("Отпустили групповую блокировку {RtqTaskGroupLock} в процессе завершения задачи {RtqTaskId}", new { RtqTaskGroupLock = taskMeta.TaskGroupLock, RtqTaskId = taskMeta.Id });
}
}
}
}
private static void DoTest(TestConfig cfg)
{
var cassandraOpTimeout = TimeSpan.FromMilliseconds(cfg.TesterConfig.CassandraClusterSettings.Timeout);
var longOpDuration = cfg.TesterConfig.LockTtl.Add(cassandraOpTimeout).Multiply(cfg.TesterConfig.CassandraClusterSettings.Attempts);
var lockIds = Enumerable.Range(0, cfg.Locks).Select(x => Guid.NewGuid().ToString()).ToArray();
var resources = new ConcurrentDictionary <string, Guid>();
var opsCounters = new ConcurrentDictionary <string, int>();
var allowFails = cfg.TesterConfig.CassandraFailProbability.HasValue;
using (var tester = new RemoteLockerTester(cfg.TesterConfig))
{
var stopSignal = new ManualResetEvent(false);
var syncSignal = new ManualResetEvent(true);
Task syncerThread = null;
if (cfg.SyncInterval.HasValue)
{
syncerThread = Task.Factory.StartNew(() =>
{
do
{
syncSignal.Reset();
Thread.Sleep(longOpDuration);
syncSignal.Set();
} while (!stopSignal.WaitOne(cfg.SyncInterval.Value));
});
}
var localTester = tester;
var actions = new Action <MultithreadingTestHelper.RunState> [cfg.TesterConfig.LockersCount];
for (var th = 0; th < actions.Length; th++)
{
var remoteLockCreator = tester[th];
actions[th] = state =>
{
var op = 0;
while (op < cfg.OperationsPerThread)
{
IRemoteLock @lock = null;
var disposed = false;
try
{
if (state.ErrorOccurred)
{
break;
}
var lockIndex = ThreadLocalRandom.Instance.Next(lockIds.Length);
var lockId = lockIds[lockIndex];
@lock = Lock(remoteLockCreator, syncSignal, lockId, state);
if (@lock == null)
{
break;
}
var localOpsCounter = opsCounters.GetOrAdd(lockId, 0);
var resource = Guid.NewGuid();
resources[lockId] = resource;
var opDuration = TimeSpan.FromMilliseconds(ThreadLocalRandom.Instance.Next(1, 47));
if (ThreadLocalRandom.Instance.NextDouble() < cfg.FastRunningOpProbability)
{
opDuration = TimeSpan.Zero;
}
else if (ThreadLocalRandom.Instance.NextDouble() < cfg.LongRunningOpProbability)
{
opDuration = opDuration.Add(longOpDuration);
}
Thread.Sleep(opDuration);
CollectionAssert.AreEqual(new[] { @lock.ThreadId }, localTester.GetThreadsInMainRow(lockId));
Assert.That(localTester.GetThreadsInShadeRow(lockId), Is.Not.Contains(@lock.ThreadId));
var lockMetadata = localTester.GetLockMetadata(lockId);
Assert.That(lockMetadata.ProbableOwnerThreadId, Is.EqualTo(@lock.ThreadId));
Assert.That(resources[lockId], Is.EqualTo(resource));
Assert.That(opsCounters[lockId], Is.EqualTo(localOpsCounter));
if (++localOpsCounter % (cfg.TesterConfig.LockersCount * cfg.OperationsPerThread / 100) == 0)
{
Console.Out.Write(".");
}
opsCounters[lockId] = localOpsCounter;
@lock.Dispose();
disposed = true;
Thread.Sleep(1);
/*
* If Cassandra can fail, then @lock.Dispose() could not delete column from Cassandra.
* But it must be automatically deleted after lockTtl.
* For performance reason we do first check manually, because .After sleeps at least polling interval before first check
*/
if (localTester.GetThreadsInMainRow(lockId).Contains(@lock.ThreadId))
{
Assert.That(() => localTester.GetThreadsInMainRow(lockId), Is.Not
.Contains(@lock.ThreadId)
.After(2 * (int)cfg.TesterConfig.LockTtl.TotalMilliseconds, 100));
}
op++;
}
catch (FailedCassandraClusterException)
{
if (!allowFails)
{
throw;
}
}
finally
{
if (@lock != null && !disposed)
{
@lock.Dispose();
}
}
}
};
}
MultithreadingTestHelper.RunOnSeparateThreads(TimeSpan.FromMinutes(30), actions);
stopSignal.Set();
syncerThread?.Wait();
Assert.That(opsCounters.Sum(x => x.Value), Is.EqualTo(cfg.TesterConfig.LockersCount * cfg.OperationsPerThread));
}
}
public bool TryGetLock(string lockId, out IRemoteLock remoteLock)
{
return(remoteLockers.Single().TryGetLock(lockId, out remoteLock));
}
