private bool TryMoveToRehab(string dbName, DatabaseTopology topology, Dictionary <string, ClusterNodeStatusReport> current, string member) { DatabaseStatusReport dbStats = null; if (current.TryGetValue(member, out var nodeStats) && nodeStats.Status == ClusterNodeStatusReport.ReportStatus.Ok && nodeStats.Report.TryGetValue(dbName, out dbStats) && dbStats.Status != Faulted) { return(false); } string reason; if (nodeStats == null) { reason = "In rehabilitation because it had no status report in the latest cluster stats"; } else if (nodeStats.Status != ClusterNodeStatusReport.ReportStatus.Ok) { reason = $"In rehabilitation because the last report status was \"{nodeStats.Status}\""; } else if (nodeStats.Report.TryGetValue(dbName, out var stats) && stats.Status == Faulted) { reason = "In rehabilitation because the DatabaseStatus for this node is Faulted"; } else { reason = "In rehabilitation because the node is reachable but had no report about the database"; } if (nodeStats?.Error != null) { reason += $". {nodeStats.Error}"; } if (dbStats?.Error != null) { reason += $". {dbStats.Error}"; } if (topology.Rehabs.Contains(member) == false) { topology.Members.Remove(member); topology.Rehabs.Add(member); } topology.DemotionReasons[member] = reason; topology.PromotablesStatus[member] = DatabasePromotionStatus.NotResponding; if (_logger.IsOperationsEnabled) { _logger.Operations(reason); } return(true); }
private static void FillReplicationInfo(DocumentDatabase dbInstance, DatabaseStatusReport report) { foreach (var outgoing in dbInstance.ReplicationLoader.OutgoingHandlers) { var node = outgoing.GetNode(); if (node != null) { report.LastSentEtag.Add(node, outgoing._lastSentDocumentEtag); } } }
private static void FillDocumentsInfo(DatabaseStatusReport prevDatabaseReport, DocumentDatabase dbInstance, DatabaseStatusReport report, DocumentsOperationContext context, DocumentsStorage documentsStorage) { if (prevDatabaseReport?.LastTransactionId != null && prevDatabaseReport.LastTransactionId == dbInstance.LastTransactionId) { report.LastEtag = prevDatabaseReport.LastEtag; report.LastTombstoneEtag = prevDatabaseReport.LastTombstoneEtag; report.NumberOfConflicts = prevDatabaseReport.NumberOfConflicts; report.NumberOfDocuments = prevDatabaseReport.NumberOfDocuments; report.DatabaseChangeVector = prevDatabaseReport.DatabaseChangeVector; } else { using (var tx = context.OpenReadTransaction()) { report.LastEtag = DocumentsStorage.ReadLastEtag(tx.InnerTransaction); report.LastTombstoneEtag = DocumentsStorage.ReadLastTombstoneEtag(tx.InnerTransaction); report.NumberOfConflicts = documentsStorage.ConflictsStorage.ConflictsCount; report.NumberOfDocuments = documentsStorage.GetNumberOfDocuments(context); report.DatabaseChangeVector = DocumentsStorage.GetDatabaseChangeVector(context); } } }
private static void FillIndexInfo(Index index, DocumentsOperationContext context, DateTime now, DatabaseStatusReport report) { var stats = index.GetIndexStats(context); var lastQueried = GetLastQueryInfo(index, now); //We might have old version of this index with the same name report.LastIndexStats[index.Name] = new DatabaseStatusReport.ObservedIndexStatus { LastIndexedEtag = stats.LastProcessedEtag, LastQueried = lastQueried, IsSideBySide = index.Name.StartsWith(Constants.Documents.Indexing.SideBySideIndexNamePrefix, StringComparison.OrdinalIgnoreCase), IsStale = stats.IsStale, State = index.State, LastTransactionId = index.LastTransactionId }; }
private static void FillClusterTransactionInfo(DatabaseStatusReport report, DocumentDatabase dbInstance) { report.LastTransactionId = dbInstance.LastTransactionId; report.LastCompletedClusterTransaction = dbInstance.LastCompletedClusterTransaction; }
private Dictionary <string, DatabaseStatusReport> CollectDatabaseInformation(TransactionOperationContext ctx, Dictionary <string, DatabaseStatusReport> prevReport) { var result = new Dictionary <string, DatabaseStatusReport>(); foreach (var dbName in _server.Cluster.GetDatabaseNames(ctx)) { if (_token.IsCancellationRequested) { return(result); } var report = new DatabaseStatusReport { Name = dbName, NodeName = _server.NodeTag }; if (_server.DatabasesLandlord.DatabasesCache.TryGetValue(dbName, out var dbTask, out var details) == false) { DatabaseTopology topology; using (var rawRecord = _server.Cluster.ReadRawDatabaseRecord(ctx, dbName)) { if (rawRecord == null) { continue; // Database does not exists in this server } topology = rawRecord.Topology; } if (topology == null) { continue; } if (topology.RelevantFor(_server.NodeTag) == false) { continue; } report.Status = DatabaseStatus.Unloaded; result[dbName] = report; continue; } report.UpTime = SystemTime.UtcNow - details.InCacheSince; if (dbTask.IsFaulted) { var extractSingleInnerException = dbTask.Exception.ExtractSingleInnerException(); if (Equals(extractSingleInnerException.Data[DatabasesLandlord.DoNotRemove], true)) { report.Status = DatabaseStatus.Unloaded; result[dbName] = report; continue; } } if (dbTask.IsCanceled || dbTask.IsFaulted) { report.Status = DatabaseStatus.Faulted; report.Error = dbTask.Exception.ToString(); result[dbName] = report; continue; } if (dbTask.IsCompleted == false) { report.Status = DatabaseStatus.Loading; result[dbName] = report; continue; } var dbInstance = dbTask.Result; var currentHash = dbInstance.GetEnvironmentsHash(); report.EnvironmentsHash = currentHash; var documentsStorage = dbInstance.DocumentsStorage; var indexStorage = dbInstance.IndexStore; if (dbInstance.DatabaseShutdown.IsCancellationRequested) { report.Status = DatabaseStatus.Shutdown; result[dbName] = report; continue; } report.Status = DatabaseStatus.Loaded; try { var now = dbInstance.Time.GetUtcNow(); FillReplicationInfo(dbInstance, report); prevReport.TryGetValue(dbName, out var prevDatabaseReport); if (SupportedFeatures.Heartbeats.SendChangesOnly && prevDatabaseReport != null && prevDatabaseReport.EnvironmentsHash == currentHash) { report.Status = DatabaseStatus.NoChange; result[dbName] = report; continue; } using (documentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext context)) { FillDocumentsInfo(prevDatabaseReport, dbInstance, report, context, documentsStorage); FillClusterTransactionInfo(report, dbInstance); if (indexStorage != null) { foreach (var index in indexStorage.GetIndexes()) { DatabaseStatusReport.ObservedIndexStatus stat = null; if (prevDatabaseReport?.LastIndexStats.TryGetValue(index.Name, out stat) == true && stat?.LastTransactionId == index.LastTransactionId) { report.LastIndexStats[index.Name] = stat; continue; } using (context.OpenReadTransaction()) { FillIndexInfo(index, context, now, report); } } } } } catch (Exception e) { report.EnvironmentsHash = 0; // on error we should do the complete report collaction path report.Error = e.ToString(); } result[dbName] = report; } return(result); }
private bool TryMoveToRehab(string dbName, DatabaseTopology topology, Dictionary <string, ClusterNodeStatusReport> current, string member) { DatabaseStatusReport dbStats = null; if (current.TryGetValue(member, out var nodeStats) && nodeStats.Status == ClusterNodeStatusReport.ReportStatus.Ok && nodeStats.Report.TryGetValue(dbName, out dbStats) && dbStats.Status != Faulted) { return(false); } string reason; if (nodeStats == null) { reason = "Node in rehabilitation due to no status report in the latest cluster stats"; } else if (nodeStats.Status != ClusterNodeStatusReport.ReportStatus.Ok) { switch (nodeStats.Status) { case ClusterNodeStatusReport.ReportStatus.Timeout: reason = $"Node in rehabilitation due to timeout reached trying to get stats from node.{Environment.NewLine}"; break; default: reason = $"Node in rehabilitation due to last report status being '{nodeStats.Status}'.{Environment.NewLine}"; break; } } else if (nodeStats.Report.TryGetValue(dbName, out var stats) && stats.Status == Faulted) { reason = $"In rehabilitation because the DatabaseStatus for this node is {nameof(Faulted)}.{Environment.NewLine}"; } else { reason = $"In rehabilitation because the node is reachable but had no report about the database.{Environment.NewLine}"; } if (nodeStats?.Error != null) { reason += $". {nodeStats.Error}"; } if (dbStats?.Error != null) { reason += $". {dbStats.Error}"; } if (topology.Rehabs.Contains(member) == false) { topology.Members.Remove(member); topology.Rehabs.Add(member); } topology.DemotionReasons[member] = reason; topology.PromotablesStatus[member] = DatabasePromotionStatus.NotResponding; if (_logger.IsOperationsEnabled) { _logger.Operations($"Node {member} of database '{dbName}': {reason}"); } return(true); }
private IEnumerable <(string name, DatabaseStatusReport report)> CollectDatabaseInformation(TransactionOperationContext ctx) { foreach (var dbName in _server.Cluster.GetDatabaseNames(ctx)) { if (_token.IsCancellationRequested) { yield break; } if (_server.DatabasesLandlord.DatabasesCache.TryGetValue(dbName, out var dbTask) == false) { continue; // Database does not exists in this server } var report = new DatabaseStatusReport { Name = dbName, NodeName = _server.NodeTag }; if (dbTask.IsCanceled || dbTask.IsFaulted) { report.Status = DatabaseStatus.Faulted; report.Error = dbTask.Exception.ToString(); yield return(dbName, report); continue; } if (dbTask.IsCompleted == false) { report.Status = DatabaseStatus.Loading; yield return(dbName, report); continue; } var dbInstance = dbTask.Result; var documentsStorage = dbInstance.DocumentsStorage; var indexStorage = dbInstance.IndexStore; if (dbInstance.DatabaseShutdown.IsCancellationRequested) { report.Status = DatabaseStatus.Shutdown; yield return(dbName, report); continue; } report.Status = DatabaseStatus.Loaded; try { using (documentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext context)) using (var tx = context.OpenReadTransaction()) { report.LastEtag = DocumentsStorage.ReadLastEtag(tx.InnerTransaction); report.LastTombstoneEtag = DocumentsStorage.ReadLastTombstoneEtag(tx.InnerTransaction); report.NumberOfConflicts = documentsStorage.ConflictsStorage.ConflictsCount; report.NumberOfDocuments = documentsStorage.GetNumberOfDocuments(context); report.LastChangeVector = DocumentsStorage.GetDatabaseChangeVector(context); if (indexStorage != null) { foreach (var index in indexStorage.GetIndexes()) { var stats = index.GetIndexStats(context); //We might have old version of this index with the same name report.LastIndexStats.Add(index.Name, new DatabaseStatusReport.ObservedIndexStatus { LastIndexedEtag = stats.LastProcessedEtag, IsSideBySide = false, // TODO: fix this so it get whatever this has side by side or not IsStale = stats.IsStale }); } } } } catch (Exception e) { report.Error = e.ToString(); } yield return(dbName, report); } }
private IEnumerable <(string name, DatabaseStatusReport report)> CollectDatabaseInformation(TransactionOperationContext ctx) { foreach (var dbName in _server.Cluster.GetDatabaseNames(ctx)) { if (_token.IsCancellationRequested) { yield break; } var report = new DatabaseStatusReport { Name = dbName, NodeName = _server.NodeTag }; if (_server.DatabasesLandlord.DatabasesCache.TryGetValue(dbName, out var dbTask) == false) { var recorod = _server.Cluster.ReadDatabase(ctx, dbName); if (recorod == null || recorod.Topology.RelevantFor(_server.NodeTag) == false) { continue; // Database does not exists in this server } report.Status = DatabaseStatus.Unloaded; yield return(dbName, report); continue; } if (dbTask.IsFaulted) { var extractSingleInnerException = dbTask.Exception.ExtractSingleInnerException(); if (Equals(extractSingleInnerException.Data[DatabasesLandlord.DoNotRemove], true)) { report.Status = DatabaseStatus.Unloaded; yield return(dbName, report); continue; } } if (dbTask.IsCanceled || dbTask.IsFaulted) { report.Status = DatabaseStatus.Faulted; report.Error = dbTask.Exception.ToString(); yield return(dbName, report); continue; } if (dbTask.IsCompleted == false) { report.Status = DatabaseStatus.Loading; yield return(dbName, report); continue; } var dbInstance = dbTask.Result; var documentsStorage = dbInstance.DocumentsStorage; var indexStorage = dbInstance.IndexStore; if (dbInstance.DatabaseShutdown.IsCancellationRequested) { report.Status = DatabaseStatus.Shutdown; yield return(dbName, report); continue; } report.Status = DatabaseStatus.Loaded; try { using (documentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext context)) using (var tx = context.OpenReadTransaction()) { report.LastEtag = DocumentsStorage.ReadLastEtag(tx.InnerTransaction); report.LastTombstoneEtag = DocumentsStorage.ReadLastTombstoneEtag(tx.InnerTransaction); report.NumberOfConflicts = documentsStorage.ConflictsStorage.ConflictsCount; report.NumberOfDocuments = documentsStorage.GetNumberOfDocuments(context); report.DatabaseChangeVector = DocumentsStorage.GetDatabaseChangeVector(context); foreach (var outgoing in dbInstance.ReplicationLoader.OutgoingHandlers) { var node = outgoing.GetNode(); if (node != null) { report.LastSentEtag.Add(node, outgoing._lastSentDocumentEtag); } } if (indexStorage != null) { foreach (var index in indexStorage.GetIndexes()) { var stats = index.GetIndexStats(context); //We might have old version of this index with the same name report.LastIndexStats[index.Name] = new DatabaseStatusReport.ObservedIndexStatus { LastIndexedEtag = stats.LastProcessedEtag, IsSideBySide = index.Name.StartsWith(Constants.Documents.Indexing.SideBySideIndexNamePrefix, StringComparison.OrdinalIgnoreCase), IsStale = stats.IsStale, State = index.State }; } } } } catch (Exception e) { report.Error = e.ToString(); } yield return(dbName, report); } }