private bool TryRelock(RemoteLockState remoteLockState) { var attempt = 1; while (true) { try { using (metrics.CassandraImplRelockOp.NewContext(remoteLockState.ToString())) { var relocked = remoteLockImplementation.TryRelock(remoteLockState.LockId, remoteLockState.ThreadId); if (!relocked) { logger.Error("Cannot relock. Possible lock metadata corruption for: {0}", remoteLockState); } return(relocked); } } catch (Exception e) { var shortSleep = ThreadLocalRandom.Instance.Next(50 * (int)Math.Exp(Math.Min(attempt++, 5))); logger.Warn(e, "remoteLockImplementation.Relock() failed for: {0}. Will sleep for {1} ms", remoteLockState, shortSleep); Thread.Sleep(shortSleep); } } }
private void KeepLockAlive(RemoteLockState remoteLockState) { TimeSpan?timeToSleep = null; lock (remoteLockState) { var nextKeepAliveMoment = remoteLockState.NextKeepAliveMoment; if (nextKeepAliveMoment == null) { return; } var utcNow = Timestamp.Now; if (utcNow < nextKeepAliveMoment) { timeToSleep = nextKeepAliveMoment - utcNow; } } if (timeToSleep.HasValue) { Thread.Sleep(timeToSleep.Value); } lock (remoteLockState) { if (remoteLockState.NextKeepAliveMoment == null) { return; } var relocked = TryRelock(remoteLockState); if (relocked && !remoteLocksQueue.IsAddingCompleted) { remoteLockState.NextKeepAliveMoment = Timestamp.Now.Add(keepLockAliveInterval); remoteLocksQueue.Add(remoteLockState); } } }
private void KeepLockAlive([NotNull] RemoteLockState remoteLockState) { TimeSpan?timeToSleep = null; lock (remoteLockState) { if (!remoteLockState.IsAlive) { return; } var utcNow = Timestamp.Now; var nextKeepAliveMoment = remoteLockState.HeartbeatMoment.Add(keepLockAliveInterval); if (utcNow < nextKeepAliveMoment) { timeToSleep = nextKeepAliveMoment - utcNow; } } if (timeToSleep.HasValue) { Thread.Sleep(timeToSleep.Value); } lock (remoteLockState) { if (!remoteLockState.IsAlive) { return; } var relocked = TryRelock(remoteLockState); if (!relocked) { remoteLockState.LockAliveTokenSource.Cancel(); return; } if (!remoteLocksQueue.IsAddingCompleted) { remoteLocksQueue.Add(remoteLockState); } } }
private IRemoteLock DoTryAcquireLock(string lockId, string threadId, out string rivalThreadId) { if (remoteLocksById.TryGetValue(lockId, out var rival)) { rivalThreadId = rival.ThreadId; return(null); } var attempt = 1; while (true) { LockAttemptResult lockAttempt; var initialHeartbeatMoment = Timestamp.Now; using (metrics.CassandraImplTryLockOp.NewContext(FormatLockOperationId(lockId, threadId))) lockAttempt = remoteLockImplementation.TryLock(lockId, threadId); switch (lockAttempt.Status) { case LockAttemptStatus.Success: rivalThreadId = null; var remoteLockState = new RemoteLockState(lockId, threadId, initialHeartbeatMoment); if (!remoteLocksById.TryAdd(lockId, remoteLockState)) { throw new InvalidOperationException($"RemoteLocker state is corrupted. lockId: {lockId}, threadId: {threadId}, remoteLocksById[lockId]: {remoteLockState}"); } remoteLocksQueue.Add(remoteLockState); return(new RemoteLockHandle(lockId, threadId, remoteLockState.LockAliveTokenSource.Token, this)); case LockAttemptStatus.AnotherThreadIsOwner: rivalThreadId = lockAttempt.OwnerId; return(null); case LockAttemptStatus.ConcurrentAttempt: var shortSleep = ThreadLocalRandom.Instance.Next(50 * (int)Math.Exp(Math.Min(attempt++, 5))); logger.Warn("remoteLockImplementation.TryLock() returned LockAttemptStatus.ConcurrentAttempt for lockId: {0}, threadId: {1}. Will sleep for {2} ms", lockId, threadId, shortSleep); Thread.Sleep(shortSleep); break; default: throw new InvalidOperationException($"Invalid LockAttemptStatus: {lockAttempt.Status}"); } } }
private void Unlock(RemoteLockState remoteLockState) { lock (remoteLockState) { remoteLockState.NextKeepAliveMoment = null; try { using (metrics.CassandraImplUnlockOp.NewContext(remoteLockState.ToString())) { if (!remoteLockImplementation.TryUnlock(remoteLockState.LockId, remoteLockState.ThreadId)) { logger.Error("Cannot unlock. Possible lock metadata corruption for: {0}", remoteLockState); } } } catch (Exception e) { logger.Error(e, "remoteLockImplementation.Unlock() failed for: {0}", remoteLockState); } } }
private bool TryRelock([NotNull] RemoteLockState remoteLockState) { var attempt = 1; while (true) { var nextHeartbeatMoment = Timestamp.Now; if (remoteLockState.HeartbeatMoment.Add(lockTtl.Multiply(0.5)) < nextHeartbeatMoment) { logger.Error("KeepLockAlive() freeze is detected on attempt #{0}. Signal LockAliveToken to prevent possible lock collision for: {1}", attempt, remoteLockState); return(false); } try { using (metrics.CassandraImplRelockOp.NewContext(remoteLockState.ToString())) { var relocked = remoteLockImplementation.TryRelock(remoteLockState.LockId, remoteLockState.ThreadId); if (relocked) { remoteLockState.HeartbeatMoment = nextHeartbeatMoment; } else { logger.Error("Cannot relock on attempt #{0}. Possible lock metadata corruption for: {1}", attempt, remoteLockState); } return(relocked); } } catch (Exception e) { var shortSleep = ThreadLocalRandom.Instance.Next(50 * (int)Math.Exp(Math.Min(attempt++, 5))); logger.Warn(e, "remoteLockImplementation.TryRelock() attempt #{0} failed for: {1}. Will sleep for {2} ms", attempt, remoteLockState, shortSleep); Thread.Sleep(shortSleep); } } }