private bool TryRelock(RemoteLockState remoteLockState)
{
var attempt = 1;
while (true)
{
try
{
using (metrics.CassandraImplRelockOp.NewContext(remoteLockState.ToString()))
{
var relocked = remoteLockImplementation.TryRelock(remoteLockState.LockId, remoteLockState.ThreadId);
if (!relocked)
{
logger.Error("Cannot relock. Possible lock metadata corruption for: {0}", remoteLockState);
}
return(relocked);
}
}
catch (Exception e)
{
var shortSleep = ThreadLocalRandom.Instance.Next(50 * (int)Math.Exp(Math.Min(attempt++, 5)));
logger.Warn(e, "remoteLockImplementation.Relock() failed for: {0}. Will sleep for {1} ms", remoteLockState, shortSleep);
Thread.Sleep(shortSleep);
}
}
}
private void KeepLockAlive(RemoteLockState remoteLockState)
{
TimeSpan?timeToSleep = null;
lock (remoteLockState)
{
var nextKeepAliveMoment = remoteLockState.NextKeepAliveMoment;
if (nextKeepAliveMoment == null)
{
return;
}
var utcNow = Timestamp.Now;
if (utcNow < nextKeepAliveMoment)
{
timeToSleep = nextKeepAliveMoment - utcNow;
}
}
if (timeToSleep.HasValue)
{
Thread.Sleep(timeToSleep.Value);
}
lock (remoteLockState)
{
if (remoteLockState.NextKeepAliveMoment == null)
{
return;
}
var relocked = TryRelock(remoteLockState);
if (relocked && !remoteLocksQueue.IsAddingCompleted)
{
remoteLockState.NextKeepAliveMoment = Timestamp.Now.Add(keepLockAliveInterval);
remoteLocksQueue.Add(remoteLockState);
}
}
}
private void KeepLockAlive([NotNull] RemoteLockState remoteLockState)
{
TimeSpan?timeToSleep = null;
lock (remoteLockState)
{
if (!remoteLockState.IsAlive)
{
return;
}
var utcNow = Timestamp.Now;
var nextKeepAliveMoment = remoteLockState.HeartbeatMoment.Add(keepLockAliveInterval);
if (utcNow < nextKeepAliveMoment)
{
timeToSleep = nextKeepAliveMoment - utcNow;
}
}
if (timeToSleep.HasValue)
{
Thread.Sleep(timeToSleep.Value);
}
lock (remoteLockState)
{
if (!remoteLockState.IsAlive)
{
return;
}
var relocked = TryRelock(remoteLockState);
if (!relocked)
{
remoteLockState.LockAliveTokenSource.Cancel();
return;
}
if (!remoteLocksQueue.IsAddingCompleted)
{
remoteLocksQueue.Add(remoteLockState);
}
}
}
private IRemoteLock DoTryAcquireLock(string lockId, string threadId, out string rivalThreadId)
{
if (remoteLocksById.TryGetValue(lockId, out var rival))
{
rivalThreadId = rival.ThreadId;
return(null);
}
var attempt = 1;
while (true)
{
LockAttemptResult lockAttempt;
var initialHeartbeatMoment = Timestamp.Now;
using (metrics.CassandraImplTryLockOp.NewContext(FormatLockOperationId(lockId, threadId)))
lockAttempt = remoteLockImplementation.TryLock(lockId, threadId);
switch (lockAttempt.Status)
{
case LockAttemptStatus.Success:
rivalThreadId = null;
var remoteLockState = new RemoteLockState(lockId, threadId, initialHeartbeatMoment);
if (!remoteLocksById.TryAdd(lockId, remoteLockState))
{
throw new InvalidOperationException($"RemoteLocker state is corrupted. lockId: {lockId}, threadId: {threadId}, remoteLocksById[lockId]: {remoteLockState}");
}
remoteLocksQueue.Add(remoteLockState);
return(new RemoteLockHandle(lockId, threadId, remoteLockState.LockAliveTokenSource.Token, this));
case LockAttemptStatus.AnotherThreadIsOwner:
rivalThreadId = lockAttempt.OwnerId;
return(null);
case LockAttemptStatus.ConcurrentAttempt:
var shortSleep = ThreadLocalRandom.Instance.Next(50 * (int)Math.Exp(Math.Min(attempt++, 5)));
logger.Warn("remoteLockImplementation.TryLock() returned LockAttemptStatus.ConcurrentAttempt for lockId: {0}, threadId: {1}. Will sleep for {2} ms", lockId, threadId, shortSleep);
Thread.Sleep(shortSleep);
break;
default:
throw new InvalidOperationException($"Invalid LockAttemptStatus: {lockAttempt.Status}");
}
}
}
private void Unlock(RemoteLockState remoteLockState)
{
lock (remoteLockState)
{
remoteLockState.NextKeepAliveMoment = null;
try
{
using (metrics.CassandraImplUnlockOp.NewContext(remoteLockState.ToString()))
{
if (!remoteLockImplementation.TryUnlock(remoteLockState.LockId, remoteLockState.ThreadId))
{
logger.Error("Cannot unlock. Possible lock metadata corruption for: {0}", remoteLockState);
}
}
}
catch (Exception e)
{
logger.Error(e, "remoteLockImplementation.Unlock() failed for: {0}", remoteLockState);
}
}
}
private bool TryRelock([NotNull] RemoteLockState remoteLockState)
{
var attempt = 1;
while (true)
{
var nextHeartbeatMoment = Timestamp.Now;
if (remoteLockState.HeartbeatMoment.Add(lockTtl.Multiply(0.5)) < nextHeartbeatMoment)
{
logger.Error("KeepLockAlive() freeze is detected on attempt #{0}. Signal LockAliveToken to prevent possible lock collision for: {1}", attempt, remoteLockState);
return(false);
}
try
{
using (metrics.CassandraImplRelockOp.NewContext(remoteLockState.ToString()))
{
var relocked = remoteLockImplementation.TryRelock(remoteLockState.LockId, remoteLockState.ThreadId);
if (relocked)
{
remoteLockState.HeartbeatMoment = nextHeartbeatMoment;
}
else
{
logger.Error("Cannot relock on attempt #{0}. Possible lock metadata corruption for: {1}", attempt, remoteLockState);
}
return(relocked);
}
}
catch (Exception e)
{
var shortSleep = ThreadLocalRandom.Instance.Next(50 * (int)Math.Exp(Math.Min(attempt++, 5)));
logger.Warn(e, "remoteLockImplementation.TryRelock() attempt #{0} failed for: {1}. Will sleep for {2} ms", attempt, remoteLockState, shortSleep);
Thread.Sleep(shortSleep);
}
}
}
