private bool TryMoveToRehab(string dbName, DatabaseTopology topology, Dictionary <string, ClusterNodeStatusReport> current, string member)
{
DatabaseStatusReport dbStats = null;
if (current.TryGetValue(member, out var nodeStats) &&
nodeStats.Status == ClusterNodeStatusReport.ReportStatus.Ok &&
nodeStats.Report.TryGetValue(dbName, out dbStats) && dbStats.Status != Faulted)
{
return(false);
}
string reason;
if (nodeStats == null)
{
reason = "In rehabilitation because it had no status report in the latest cluster stats";
}
else if (nodeStats.Status != ClusterNodeStatusReport.ReportStatus.Ok)
{
reason = $"In rehabilitation because the last report status was \"{nodeStats.Status}\"";
}
else if (nodeStats.Report.TryGetValue(dbName, out var stats) && stats.Status == Faulted)
{
reason = "In rehabilitation because the DatabaseStatus for this node is Faulted";
}
else
{
reason = "In rehabilitation because the node is reachable but had no report about the database";
}
if (nodeStats?.Error != null)
{
reason += $". {nodeStats.Error}";
}
if (dbStats?.Error != null)
{
reason += $". {dbStats.Error}";
}
if (topology.Rehabs.Contains(member) == false)
{
topology.Members.Remove(member);
topology.Rehabs.Add(member);
}
topology.DemotionReasons[member] = reason;
topology.PromotablesStatus[member] = DatabasePromotionStatus.NotResponding;
if (_logger.IsOperationsEnabled)
{
_logger.Operations(reason);
}
return(true);
}
private static void FillReplicationInfo(DocumentDatabase dbInstance, DatabaseStatusReport report)
{
foreach (var outgoing in dbInstance.ReplicationLoader.OutgoingHandlers)
{
var node = outgoing.GetNode();
if (node != null)
{
report.LastSentEtag.Add(node, outgoing._lastSentDocumentEtag);
}
}
}
private static void FillDocumentsInfo(DatabaseStatusReport prevDatabaseReport, DocumentDatabase dbInstance, DatabaseStatusReport report,
DocumentsOperationContext context, DocumentsStorage documentsStorage)
{
if (prevDatabaseReport?.LastTransactionId != null && prevDatabaseReport.LastTransactionId == dbInstance.LastTransactionId)
{
report.LastEtag = prevDatabaseReport.LastEtag;
report.LastTombstoneEtag = prevDatabaseReport.LastTombstoneEtag;
report.NumberOfConflicts = prevDatabaseReport.NumberOfConflicts;
report.NumberOfDocuments = prevDatabaseReport.NumberOfDocuments;
report.DatabaseChangeVector = prevDatabaseReport.DatabaseChangeVector;
}
else
{
using (var tx = context.OpenReadTransaction())
{
report.LastEtag = DocumentsStorage.ReadLastEtag(tx.InnerTransaction);
report.LastTombstoneEtag = DocumentsStorage.ReadLastTombstoneEtag(tx.InnerTransaction);
report.NumberOfConflicts = documentsStorage.ConflictsStorage.ConflictsCount;
report.NumberOfDocuments = documentsStorage.GetNumberOfDocuments(context);
report.DatabaseChangeVector = DocumentsStorage.GetDatabaseChangeVector(context);
}
}
}
private static void FillIndexInfo(Index index, DocumentsOperationContext context, DateTime now, DatabaseStatusReport report)
{
var stats = index.GetIndexStats(context);
var lastQueried = GetLastQueryInfo(index, now);
//We might have old version of this index with the same name
report.LastIndexStats[index.Name] = new DatabaseStatusReport.ObservedIndexStatus
{
LastIndexedEtag = stats.LastProcessedEtag,
LastQueried = lastQueried,
IsSideBySide = index.Name.StartsWith(Constants.Documents.Indexing.SideBySideIndexNamePrefix, StringComparison.OrdinalIgnoreCase),
IsStale = stats.IsStale,
State = index.State,
LastTransactionId = index.LastTransactionId
};
}
private static void FillClusterTransactionInfo(DatabaseStatusReport report, DocumentDatabase dbInstance)
{
report.LastTransactionId = dbInstance.LastTransactionId;
report.LastCompletedClusterTransaction = dbInstance.LastCompletedClusterTransaction;
}
private Dictionary <string, DatabaseStatusReport> CollectDatabaseInformation(TransactionOperationContext ctx, Dictionary <string, DatabaseStatusReport> prevReport)
{
var result = new Dictionary <string, DatabaseStatusReport>();
foreach (var dbName in _server.Cluster.GetDatabaseNames(ctx))
{
if (_token.IsCancellationRequested)
{
return(result);
}
var report = new DatabaseStatusReport
{
Name = dbName,
NodeName = _server.NodeTag
};
if (_server.DatabasesLandlord.DatabasesCache.TryGetValue(dbName, out var dbTask, out var details) == false)
{
DatabaseTopology topology;
using (var rawRecord = _server.Cluster.ReadRawDatabaseRecord(ctx, dbName))
{
if (rawRecord == null)
{
continue; // Database does not exists in this server
}
topology = rawRecord.Topology;
}
if (topology == null)
{
continue;
}
if (topology.RelevantFor(_server.NodeTag) == false)
{
continue;
}
report.Status = DatabaseStatus.Unloaded;
result[dbName] = report;
continue;
}
report.UpTime = SystemTime.UtcNow - details.InCacheSince;
if (dbTask.IsFaulted)
{
var extractSingleInnerException = dbTask.Exception.ExtractSingleInnerException();
if (Equals(extractSingleInnerException.Data[DatabasesLandlord.DoNotRemove], true))
{
report.Status = DatabaseStatus.Unloaded;
result[dbName] = report;
continue;
}
}
if (dbTask.IsCanceled || dbTask.IsFaulted)
{
report.Status = DatabaseStatus.Faulted;
report.Error = dbTask.Exception.ToString();
result[dbName] = report;
continue;
}
if (dbTask.IsCompleted == false)
{
report.Status = DatabaseStatus.Loading;
result[dbName] = report;
continue;
}
var dbInstance = dbTask.Result;
var currentHash = dbInstance.GetEnvironmentsHash();
report.EnvironmentsHash = currentHash;
var documentsStorage = dbInstance.DocumentsStorage;
var indexStorage = dbInstance.IndexStore;
if (dbInstance.DatabaseShutdown.IsCancellationRequested)
{
report.Status = DatabaseStatus.Shutdown;
result[dbName] = report;
continue;
}
report.Status = DatabaseStatus.Loaded;
try
{
var now = dbInstance.Time.GetUtcNow();
FillReplicationInfo(dbInstance, report);
prevReport.TryGetValue(dbName, out var prevDatabaseReport);
if (SupportedFeatures.Heartbeats.SendChangesOnly &&
prevDatabaseReport != null && prevDatabaseReport.EnvironmentsHash == currentHash)
{
report.Status = DatabaseStatus.NoChange;
result[dbName] = report;
continue;
}
using (documentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext context))
{
FillDocumentsInfo(prevDatabaseReport, dbInstance, report, context, documentsStorage);
FillClusterTransactionInfo(report, dbInstance);
if (indexStorage != null)
{
foreach (var index in indexStorage.GetIndexes())
{
DatabaseStatusReport.ObservedIndexStatus stat = null;
if (prevDatabaseReport?.LastIndexStats.TryGetValue(index.Name, out stat) == true && stat?.LastTransactionId == index.LastTransactionId)
{
report.LastIndexStats[index.Name] = stat;
continue;
}
using (context.OpenReadTransaction())
{
FillIndexInfo(index, context, now, report);
}
}
}
}
}
catch (Exception e)
{
report.EnvironmentsHash = 0; // on error we should do the complete report collaction path
report.Error = e.ToString();
}
result[dbName] = report;
}
return(result);
}
private bool TryMoveToRehab(string dbName, DatabaseTopology topology, Dictionary <string, ClusterNodeStatusReport> current, string member)
{
DatabaseStatusReport dbStats = null;
if (current.TryGetValue(member, out var nodeStats) &&
nodeStats.Status == ClusterNodeStatusReport.ReportStatus.Ok &&
nodeStats.Report.TryGetValue(dbName, out dbStats) && dbStats.Status != Faulted)
{
return(false);
}
string reason;
if (nodeStats == null)
{
reason = "Node in rehabilitation due to no status report in the latest cluster stats";
}
else if (nodeStats.Status != ClusterNodeStatusReport.ReportStatus.Ok)
{
switch (nodeStats.Status)
{
case ClusterNodeStatusReport.ReportStatus.Timeout:
reason = $"Node in rehabilitation due to timeout reached trying to get stats from node.{Environment.NewLine}";
break;
default:
reason = $"Node in rehabilitation due to last report status being '{nodeStats.Status}'.{Environment.NewLine}";
break;
}
}
else if (nodeStats.Report.TryGetValue(dbName, out var stats) && stats.Status == Faulted)
{
reason = $"In rehabilitation because the DatabaseStatus for this node is {nameof(Faulted)}.{Environment.NewLine}";
}
else
{
reason = $"In rehabilitation because the node is reachable but had no report about the database.{Environment.NewLine}";
}
if (nodeStats?.Error != null)
{
reason += $". {nodeStats.Error}";
}
if (dbStats?.Error != null)
{
reason += $". {dbStats.Error}";
}
if (topology.Rehabs.Contains(member) == false)
{
topology.Members.Remove(member);
topology.Rehabs.Add(member);
}
topology.DemotionReasons[member] = reason;
topology.PromotablesStatus[member] = DatabasePromotionStatus.NotResponding;
if (_logger.IsOperationsEnabled)
{
_logger.Operations($"Node {member} of database '{dbName}': {reason}");
}
return(true);
}
private IEnumerable <(string name, DatabaseStatusReport report)> CollectDatabaseInformation(TransactionOperationContext ctx)
{
foreach (var dbName in _server.Cluster.GetDatabaseNames(ctx))
{
if (_token.IsCancellationRequested)
{
yield break;
}
if (_server.DatabasesLandlord.DatabasesCache.TryGetValue(dbName, out var dbTask) == false)
{
continue; // Database does not exists in this server
}
var report = new DatabaseStatusReport
{
Name = dbName,
NodeName = _server.NodeTag
};
if (dbTask.IsCanceled || dbTask.IsFaulted)
{
report.Status = DatabaseStatus.Faulted;
report.Error = dbTask.Exception.ToString();
yield return(dbName, report);
continue;
}
if (dbTask.IsCompleted == false)
{
report.Status = DatabaseStatus.Loading;
yield return(dbName, report);
continue;
}
var dbInstance = dbTask.Result;
var documentsStorage = dbInstance.DocumentsStorage;
var indexStorage = dbInstance.IndexStore;
if (dbInstance.DatabaseShutdown.IsCancellationRequested)
{
report.Status = DatabaseStatus.Shutdown;
yield return(dbName, report);
continue;
}
report.Status = DatabaseStatus.Loaded;
try
{
using (documentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext context))
using (var tx = context.OpenReadTransaction())
{
report.LastEtag = DocumentsStorage.ReadLastEtag(tx.InnerTransaction);
report.LastTombstoneEtag = DocumentsStorage.ReadLastTombstoneEtag(tx.InnerTransaction);
report.NumberOfConflicts = documentsStorage.ConflictsStorage.ConflictsCount;
report.NumberOfDocuments = documentsStorage.GetNumberOfDocuments(context);
report.LastChangeVector = DocumentsStorage.GetDatabaseChangeVector(context);
if (indexStorage != null)
{
foreach (var index in indexStorage.GetIndexes())
{
var stats = index.GetIndexStats(context);
//We might have old version of this index with the same name
report.LastIndexStats.Add(index.Name, new DatabaseStatusReport.ObservedIndexStatus
{
LastIndexedEtag = stats.LastProcessedEtag,
IsSideBySide = false, // TODO: fix this so it get whatever this has side by side or not
IsStale = stats.IsStale
});
}
}
}
}
catch (Exception e)
{
report.Error = e.ToString();
}
yield return(dbName, report);
}
}
private IEnumerable <(string name, DatabaseStatusReport report)> CollectDatabaseInformation(TransactionOperationContext ctx)
{
foreach (var dbName in _server.Cluster.GetDatabaseNames(ctx))
{
if (_token.IsCancellationRequested)
{
yield break;
}
var report = new DatabaseStatusReport
{
Name = dbName,
NodeName = _server.NodeTag
};
if (_server.DatabasesLandlord.DatabasesCache.TryGetValue(dbName, out var dbTask) == false)
{
var recorod = _server.Cluster.ReadDatabase(ctx, dbName);
if (recorod == null || recorod.Topology.RelevantFor(_server.NodeTag) == false)
{
continue; // Database does not exists in this server
}
report.Status = DatabaseStatus.Unloaded;
yield return(dbName, report);
continue;
}
if (dbTask.IsFaulted)
{
var extractSingleInnerException = dbTask.Exception.ExtractSingleInnerException();
if (Equals(extractSingleInnerException.Data[DatabasesLandlord.DoNotRemove], true))
{
report.Status = DatabaseStatus.Unloaded;
yield return(dbName, report);
continue;
}
}
if (dbTask.IsCanceled || dbTask.IsFaulted)
{
report.Status = DatabaseStatus.Faulted;
report.Error = dbTask.Exception.ToString();
yield return(dbName, report);
continue;
}
if (dbTask.IsCompleted == false)
{
report.Status = DatabaseStatus.Loading;
yield return(dbName, report);
continue;
}
var dbInstance = dbTask.Result;
var documentsStorage = dbInstance.DocumentsStorage;
var indexStorage = dbInstance.IndexStore;
if (dbInstance.DatabaseShutdown.IsCancellationRequested)
{
report.Status = DatabaseStatus.Shutdown;
yield return(dbName, report);
continue;
}
report.Status = DatabaseStatus.Loaded;
try
{
using (documentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext context))
using (var tx = context.OpenReadTransaction())
{
report.LastEtag = DocumentsStorage.ReadLastEtag(tx.InnerTransaction);
report.LastTombstoneEtag = DocumentsStorage.ReadLastTombstoneEtag(tx.InnerTransaction);
report.NumberOfConflicts = documentsStorage.ConflictsStorage.ConflictsCount;
report.NumberOfDocuments = documentsStorage.GetNumberOfDocuments(context);
report.DatabaseChangeVector = DocumentsStorage.GetDatabaseChangeVector(context);
foreach (var outgoing in dbInstance.ReplicationLoader.OutgoingHandlers)
{
var node = outgoing.GetNode();
if (node != null)
{
report.LastSentEtag.Add(node, outgoing._lastSentDocumentEtag);
}
}
if (indexStorage != null)
{
foreach (var index in indexStorage.GetIndexes())
{
var stats = index.GetIndexStats(context);
//We might have old version of this index with the same name
report.LastIndexStats[index.Name] = new DatabaseStatusReport.ObservedIndexStatus
{
LastIndexedEtag = stats.LastProcessedEtag,
IsSideBySide = index.Name.StartsWith(Constants.Documents.Indexing.SideBySideIndexNamePrefix, StringComparison.OrdinalIgnoreCase),
IsStale = stats.IsStale,
State = index.State
};
}
}
}
}
catch (Exception e)
{
report.Error = e.ToString();
}
yield return(dbName, report);
}
}