private void UpdateReplicationPosition()
{
var replicationCp = _replicationCheckpoint.Read();
var writerCp = _writerCheckpoint.Read();
if (writerCp <= replicationCp)
{
return;
}
var minReplicas = _quorumSize - 1; //total - leader = min replicas
if (minReplicas == 0)
{
_replicationCheckpoint.Write(writerCp);
_replicationCheckpoint.Flush();
_replicationChange.Set();
return;
}
long[] positions;
lock (_replicaLogPositions) {
positions = _replicaLogPositions.Values.ToArray();
}
if (positions.Length < minReplicas)
{
return;
}
Array.Sort(positions);
var furthestReplicatedPosition = positions[^ minReplicas];
private void EnqueueReplicatedCommits()
{
var replicated = new List <StorageMessage.CommitAck>();
lock (_commitAcks) {
if (_commitAcks.Count > 0)
{
do
{
var ack = _commitAcks.Values[0];
if (ack.LogPosition > _replicationCheckpoint.Read())
{
break;
}
replicated.Add(ack);
_commitAcks.RemoveAt(0);
} while (_commitAcks.Count > 0);
}
}
foreach (var ack in replicated)
{
#if DEBUG
_queueStats.Enqueued();
#endif
_replicatedQueue.Enqueue(ack);
_addMsgSignal.Set();
}
}
protected override MemberInfo GetInitialMe()
{
var lastEpoch = _epochManager.GetLastEpoch();
var initialState = _memberInfo.IsReadOnlyReplica ? VNodeState.ReadOnlyLeaderless : VNodeState.Unknown;
return(MemberInfo.ForVNode(_memberInfo.InstanceId,
_timeProvider.UtcNow,
initialState,
true,
_memberInfo.InternalTcpEndPoint,
_memberInfo.InternalSecureTcpEndPoint,
_memberInfo.ExternalTcpEndPoint,
_memberInfo.ExternalSecureTcpEndPoint,
_memberInfo.HttpEndPoint,
_memberInfo.AdvertiseHostToClientAs,
_memberInfo.AdvertiseHttpPortToClientAs,
_memberInfo.AdvertiseTcpPortToClientAs,
_getLastCommitPosition(),
_writerCheckpoint.Read(),
_chaserCheckpoint.Read(),
lastEpoch == null ? -1 : lastEpoch.EpochPosition,
lastEpoch == null ? -1 : lastEpoch.EpochNumber,
lastEpoch == null ? Guid.Empty : lastEpoch.EpochId,
_nodePriority,
_memberInfo.IsReadOnlyReplica));
}
private void ConsiderBecomingLeader(Message received, bool log)
{
var replicationCurrent = _replicationCheckpoint.Read();
var indexCurrent = _indexCheckpoint.Read();
var replicationDone = replicationCurrent >= _replicationCheckpointTarget;
var indexDone = indexCurrent >= _indexCheckpointTarget;
if (log || (replicationDone && indexDone))
{
LogExtraInformation(
"Replication {progress}: {current:N0}/{target:N0}. Remaining: {remaining:N0}.",
replicationDone ? "DONE" : "IN PROGRESS",
replicationCurrent,
_replicationCheckpointTarget,
Math.Max(0, _replicationCheckpointTarget - replicationCurrent));
LogExtraInformation(
"Index {progress}: {current:N0}/{target:N0}. Remaining: {remaining:N0}.",
indexDone ? "DONE" : "IN PROGRESS",
indexCurrent,
_indexCheckpointTarget,
Math.Max(0, _indexCheckpointTarget - indexCurrent));
}
if (replicationDone && indexDone)
{
// transition to leader!
Respond(
received,
new SystemMessage.BecomeLeader(_stateCorrelationId),
"Correlation id {currentCorrelationId}",
_stateCorrelationId);
_managerState = ManagerState.Initial;
}
}
ReadAllEventsForward(ClientMessage.ReadAllEventsForward msg)
{
using (HistogramService.Measure(ReaderAllRangeHistogram)) {
var pos = new TFPos(msg.CommitPosition, msg.PreparePosition);
var lastIndexedPosition = _readIndex.LastIndexedPosition;
try {
if (msg.MaxCount > MaxPageSize)
{
throw new ArgumentException($"Read size too big, should be less than {MaxPageSize} items");
}
if (pos == TFPos.HeadOfTf)
{
var checkpoint = _writerCheckpoint.Read();
pos = new TFPos(checkpoint, checkpoint);
}
if (pos.CommitPosition < 0 || pos.PreparePosition < 0)
{
return(NoData(msg, ReadAllResult.Error, pos, lastIndexedPosition, "Invalid position."));
}
if (msg.ValidationTfLastCommitPosition == lastIndexedPosition)
{
return(NoData(msg, ReadAllResult.NotModified, pos, lastIndexedPosition));
}
var res = _readIndex.ReadAllEventsForward(pos, msg.MaxCount);
var resolved = ResolveReadAllResult(res.Records, msg.ResolveLinkTos, msg.User);
if (resolved == null)
{
return(NoData(msg, ReadAllResult.AccessDenied, pos, lastIndexedPosition));
}
var metadata = _readIndex.GetStreamMetadata(SystemStreams.AllStream);
return(new ClientMessage.ReadAllEventsForwardCompleted(
msg.CorrelationId, ReadAllResult.Success, null, resolved, metadata, false, msg.MaxCount,
res.CurrentPos, res.NextPos, res.PrevPos, lastIndexedPosition));
} catch (Exception exc) {
Log.Error(exc, "Error during processing ReadAllEventsForward request.");
return(NoData(msg, ReadAllResult.Error, pos, lastIndexedPosition, exc.Message));
}
}
}
private SeqReadResult TryReadNextInternal(int retries)
{
while (true)
{
var pos = _curPos;
var writerChk = _writerCheckpoint.Read();
if (pos >= writerChk)
{
return(SeqReadResult.Failure);
}
var chunk = _db.Manager.GetChunkFor(pos);
RecordReadResult result;
try {
result = chunk.TryReadClosestForward(chunk.ChunkHeader.GetLocalLogPosition(pos));
CountRead(chunk.IsCached);
} catch (FileBeingDeletedException) {
if (retries > MaxRetries)
{
throw new Exception(
string.Format(
"Got a file that was being deleted {0} times from TFChunkDb, likely a bug there.",
MaxRetries));
}
return(TryReadNextInternal(retries + 1));
}
if (result.Success)
{
_curPos = chunk.ChunkHeader.ChunkStartPosition + result.NextPosition;
var postPos =
result.LogRecord.GetNextLogPosition(result.LogRecord.LogPosition, result.RecordLength);
var eof = postPos == writerChk;
return(new SeqReadResult(
true, eof, result.LogRecord, result.RecordLength, result.LogRecord.LogPosition, postPos));
}
// we are the end of chunk
_curPos = chunk.ChunkHeader.ChunkEndPosition; // the start of next physical chunk
}
}
public void WaitIdle(bool waitForCheckpoints = true, bool waitForNonEmptyTf = false,
int multiplier = 1)
{
var counter = 0;
var singlePass = false;
do
{
singlePass = true;
foreach (var kvp in _queueStatsCollectors)
{
var queueStatsCollector = kvp.Key;
while (!queueStatsCollector.IsIdle())
{
Console.WriteLine($"Waiting for IDLE state for queue {queueStatsCollector.Name}...");
counter++;
singlePass = false;
if (counter > 150 * multiplier)
{
throw new ApplicationException($"Infinite WaitIdle() loop for queue: {queueStatsCollector.Name}?");
}
Thread.Sleep(100);
}
}
} while (!singlePass);
var successes = 0;
do
{
if ((waitForCheckpoints && AreCheckpointsDifferent()) ||
(waitForNonEmptyTf && _writerCheckpoint.Read() == 0))
{
Console.WriteLine("Waiting for IDLE state on checkpoints...");
counter++;
if (counter > 150 * multiplier)
{
throw new ApplicationException("Infinite WaitIdle() loop on checkpoints?");
}
Thread.Sleep(100);
}
else
{
successes++;
}
} while (successes < 2);
}
public IDictionary <string, object> GetSystemStats()
{
var stats = new Dictionary <string, object>();
GetPerfCounterInformation(stats, 0);
var process = Process.GetCurrentProcess();
var diskIo = DiskIo.GetDiskIo(process.Id, _log);
if (diskIo != null)
{
stats["proc-diskIo-readBytes"] = diskIo.ReadBytes;
stats["proc-diskIo-writtenBytes"] = diskIo.WrittenBytes;
stats["proc-diskIo-readOps"] = diskIo.ReadOps;
stats["proc-diskIo-writeOps"] = diskIo.WriteOps;
}
var tcp = TcpConnectionMonitor.Default.GetTcpStats();
stats["proc-tcp-connections"] = tcp.Connections;
stats["proc-tcp-receivingSpeed"] = tcp.ReceivingSpeed;
stats["proc-tcp-sendingSpeed"] = tcp.SendingSpeed;
stats["proc-tcp-inSend"] = tcp.InSend;
stats["proc-tcp-measureTime"] = tcp.MeasureTime;
stats["proc-tcp-pendingReceived"] = tcp.PendingReceived;
stats["proc-tcp-pendingSend"] = tcp.PendingSend;
stats["proc-tcp-receivedBytesSinceLastRun"] = tcp.ReceivedBytesSinceLastRun;
stats["proc-tcp-receivedBytesTotal"] = tcp.ReceivedBytesTotal;
stats["proc-tcp-sentBytesSinceLastRun"] = tcp.SentBytesSinceLastRun;
stats["proc-tcp-sentBytesTotal"] = tcp.SentBytesTotal;
stats["es-checksum"] = _writerCheckpoint.Read();
stats["es-checksumNonFlushed"] = _writerCheckpoint.ReadNonFlushed();
var drive = EsDriveInfo.FromDirectory(_dbPath, _log);
if (drive != null)
{
Func <string, string, string> driveStat = (diskName, stat) =>
string.Format("sys-drive-{0}-{1}", diskName.Replace("\\", "").Replace(":", ""), stat);
stats[driveStat(drive.DiskName, "availableBytes")] = drive.AvailableBytes;
stats[driveStat(drive.DiskName, "totalBytes")] = drive.TotalBytes;
stats[driveStat(drive.DiskName, "usage")] = drive.Usage;
stats[driveStat(drive.DiskName, "usedBytes")] = drive.UsedBytes;
}
Func <string, string, string> queueStat = (queueName, stat) =>
string.Format("es-queue-{0}-{1}", queueName, stat);
var queues = QueueMonitor.Default.GetStats();
foreach (var queue in queues)
{
stats[queueStat(queue.Name, "queueName")] = queue.Name;
stats[queueStat(queue.Name, "groupName")] = queue.GroupName ?? string.Empty;
stats[queueStat(queue.Name, "avgItemsPerSecond")] = queue.AvgItemsPerSecond;
stats[queueStat(queue.Name, "avgProcessingTime")] = queue.AvgProcessingTime;
stats[queueStat(queue.Name, "currentIdleTime")] = queue.CurrentIdleTime.HasValue
? queue.CurrentIdleTime.Value.ToString("G", CultureInfo.InvariantCulture)
: null;
stats[queueStat(queue.Name, "currentItemProcessingTime")] = queue.CurrentItemProcessingTime.HasValue
? queue.CurrentItemProcessingTime.Value.ToString("G", CultureInfo.InvariantCulture)
: null;
stats[queueStat(queue.Name, "idleTimePercent")] = queue.IdleTimePercent;
stats[queueStat(queue.Name, "length")] = queue.Length;
stats[queueStat(queue.Name, "lengthCurrentTryPeak")] = queue.LengthCurrentTryPeak;
stats[queueStat(queue.Name, "lengthLifetimePeak")] = queue.LengthLifetimePeak;
stats[queueStat(queue.Name, "totalItemsProcessed")] = queue.TotalItemsProcessed;
stats[queueStat(queue.Name, "inProgressMessage")] = queue.InProgressMessageType != null
? queue.InProgressMessageType.Name
: "<none>";
stats[queueStat(queue.Name, "lastProcessedMessage")] = queue.LastProcessedMessageType != null
? queue.LastProcessedMessageType.Name
: "<none>";
}
return(stats);
}
private bool AreCheckpointsDifferent()
{
return(_writerCheckpoint != null && _chaserCheckpoint != null && _writerCheckpoint.ReadNonFlushed() != _chaserCheckpoint.Read());
}
private bool IsTfEof(long postPosition)
{
return(postPosition == _writerCheckpoint.Read());
}