public Task DetailedStats()
{
using (var context = QueryOperationContext.Allocate(Database, needsServerContext: true))
{
var stats = new DetailedDatabaseStatistics();
FillDatabaseStatistics(stats, context);
using (ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext serverContext))
using (serverContext.OpenReadTransaction())
{
stats.CountOfIdentities = ServerStore.Cluster.GetNumberOfIdentities(serverContext, Database.Name);
stats.CountOfCompareExchange = ServerStore.Cluster.GetNumberOfCompareExchange(serverContext, Database.Name);
stats.CountOfCompareExchangeTombstones = ServerStore.Cluster.GetNumberOfCompareExchangeTombstones(serverContext, Database.Name);
}
using (var writer = new BlittableJsonTextWriter(context.Documents, ResponseBodyStream()))
writer.WriteDetailedDatabaseStatistics(context.Documents, stats);
}
return(Task.CompletedTask);
}
public async Task Patch()
{
var queryContext = QueryOperationContext.Allocate(Database); // we don't dispose this as operation is async
try
{
var reader = await queryContext.Documents.ReadForMemoryAsync(RequestBodyStream(), "queries/patch");
if (reader == null)
{
throw new BadRequestException("Missing JSON content.");
}
if (reader.TryGet("Query", out BlittableJsonReaderObject queryJson) == false || queryJson == null)
{
throw new BadRequestException("Missing 'Query' property.");
}
var query = IndexQueryServerSide.Create(HttpContext, queryJson, Database.QueryMetadataCache, null, queryType: QueryType.Update);
query.DisableAutoIndexCreation = GetBoolValueQueryString("disableAutoIndexCreation", false) ?? false;
if (TrafficWatchManager.HasRegisteredClients)
{
TrafficWatchQuery(query);
}
var patch = new PatchRequest(query.Metadata.GetUpdateBody(query.QueryParameters), PatchRequestType.Patch, query.Metadata.DeclaredFunctions);
await ExecuteQueryOperation(query,
(runner, options, onProgress, token) => runner.ExecutePatchQuery(
query, options, patch, query.QueryParameters, queryContext, onProgress, token),
queryContext, Operations.Operations.OperationType.UpdateByQuery);
}
catch
{
queryContext.Dispose();
throw;
}
}
public async Task StayInRehabWithErrorIndexes2()
{
var(node, leader) = await CreateRaftCluster(3, watcherCluster : true);
var database = GetDatabaseName();
await CreateDatabaseInClusterInner(new DatabaseRecord(database), 3, leader.WebUrl, null);
using (var store = new DocumentStore
{
Database = database,
Urls = new[] { leader.WebUrl }
}.Initialize())
{
var documentDatabase = await Servers[2].ServerStore.DatabasesLandlord.TryGetOrCreateResourceStore(database);
var result = store.Maintenance.Send(new PutIndexesOperation(new[] { new IndexDefinition
{
Maps = { "from user in docs.Users select new { user.Name }" },
Name = "MyIndex"
} }));
var indexResult = result[0];
await WaitForRaftIndexToBeAppliedInCluster(indexResult.RaftCommandIndex, TimeSpan.FromSeconds(15));
var index = documentDatabase.IndexStore.GetIndex("MyIndex");
index.SetState(IndexState.Error);
using (var session = store.OpenSession())
{
session.Store(new User()
{
Name = "Toli"
}, "user/1");
session.SaveChanges();
}
using (var context = QueryOperationContext.Allocate(documentDatabase, index))
using (context.OpenReadTransaction())
{
var state = index.GetIndexingState(context);
Assert.Equal(0, state.LastProcessedEtag);
}
var record = await store.Maintenance.Server.SendAsync(new GetDatabaseRecordOperation(store.Database));
record.Topology.Members.Remove(Servers[2].ServerStore.NodeTag);
record.Topology.Rehabs.Add(Servers[2].ServerStore.NodeTag);
await store.Maintenance.Server.SendAsync(new UpdateDatabaseOperation(record, record.Etag));
var rehabs = await WaitForValueAsync(async() => await GetRehabCount(store, store.Database), 1);
Assert.Equal(1, rehabs);
var val = await WaitForValueAsync(async() => await GetMembersCount(store, database), 2);
Assert.Equal(2, val);
val = await WaitForValueAsync(async() => await GetMembersCount(store, store.Database), 3);
Assert.Equal(2, val);
}
}
protected override async Task DoWork()
{
await WaitOrThrowOperationCanceled(_notificationCenter.Options.DatabaseStatsThrottle);
Stats current;
DateTime?lastIndexingErrorTime = null;
var indexes = _database.IndexStore.GetIndexes().ToList();
var needsServerContext = indexes.Any(x => x.Definition.HasCompareExchange);
var staleIndexes = 0;
var countOfIndexingErrors = 0L;
using (var context = QueryOperationContext.Allocate(_database, needsServerContext))
using (context.OpenReadTransaction())
{
// ReSharper disable once LoopCanBeConvertedToQuery
foreach (var index in indexes)
{
if (index.IsStale(context))
{
staleIndexes++;
}
var errorCount = index.GetErrorCount();
if (errorCount > 0)
{
var lastError = index.GetLastIndexingErrorTime();
if (lastError != null)
{
if (lastIndexingErrorTime == null || lastError > lastIndexingErrorTime)
{
lastIndexingErrorTime = lastError;
}
}
}
countOfIndexingErrors += errorCount;
}
current = new Stats
{
CountOfConflicts = _database.DocumentsStorage.ConflictsStorage.GetNumberOfDocumentsConflicts(context.Documents),
CountOfDocuments = _database.DocumentsStorage.GetNumberOfDocuments(context.Documents),
CountOfIndexes = indexes.Count,
CountOfStaleIndexes = staleIndexes,
CountOfIndexingErrors = countOfIndexingErrors,
LastEtag = DocumentsStorage.ReadLastEtag(context.Documents.Transaction.InnerTransaction),
GlobalChangeVector = DocumentsStorage.GetDatabaseChangeVector(context.Documents)
};
current.Collections = _database.DocumentsStorage.GetCollections(context.Documents)
.ToDictionary(x => x.Name, x => new DatabaseStatsChanged.ModifiedCollection(x.Name, x.Count, _database.DocumentsStorage.GetLastDocumentChangeVector(context.Documents.Transaction.InnerTransaction, context.Documents, x.Name)));
}
if (_latest != null && _latest.Equals(current))
{
return;
}
var modifiedCollections = _latest == null?current.Collections.Values.ToList() : ExtractModifiedCollections(current);
_notificationCenter.Add(DatabaseStatsChanged.Create(
_database.Name,
current.CountOfConflicts,
current.CountOfDocuments,
current.CountOfIndexes,
current.CountOfStaleIndexes,
current.GlobalChangeVector,
current.LastEtag,
current.CountOfIndexingErrors,
lastIndexingErrorTime,
modifiedCollections));
_latest = current;
}
public async Task NotInRehabWithDisabledIndexes3()
{
var(node, leader) = await CreateRaftCluster(3, watcherCluster : true);
var database = GetDatabaseName();
await CreateDatabaseInClusterInner(new DatabaseRecord(database), 3, leader.WebUrl, null);
using (var store = new DocumentStore
{
Database = database,
Urls = new[] { leader.WebUrl }
}.Initialize())
{
var documentDatabase = await Servers[2].ServerStore.DatabasesLandlord.TryGetOrCreateResourceStore(database);
var result = store.Maintenance.Send(new PutIndexesOperation(new[] { new IndexDefinition
{
Maps = { "from user in docs.Users select new { user.Name }" },
Name = "MyIndex"
} }));
var indexResult = result[0];
await Cluster.WaitForRaftIndexToBeAppliedInClusterAsync(indexResult.RaftCommandIndex, TimeSpan.FromSeconds(15));
await RollingIndexesClusterTests.WaitForRollingIndex(database, "MyIndex", Servers[2]);
var index = documentDatabase.IndexStore.GetIndex("MyIndex");
index.SetState(IndexState.Error);
using (var session = store.OpenSession())
{
session.Store(new User()
{
Name = "Toli"
}, "user/1");
session.SaveChanges();
}
index.SetState(IndexState.Disabled);
using (var context = QueryOperationContext.Allocate(documentDatabase, index))
using (context.OpenReadTransaction())
{
var state = index.GetIndexingState(context);
Assert.Equal(0, state.LastProcessedEtag);
}
var record = await store.Maintenance.Server.SendAsync(new GetDatabaseRecordOperation(store.Database));
record.Topology.Members.Remove(Servers[2].ServerStore.NodeTag);
record.Topology.Rehabs.Add(Servers[2].ServerStore.NodeTag);
await store.Maintenance.Server.SendAsync(new UpdateDatabaseOperation(record, record.Etag));
await WaitAndAssertForValueAsync(async() => await RavenDB_15588.GetMembersAndRehabsCount(store), expectedVal : (MembersCount : 2, RehabsCount : 1));
// assert that Servers[2] is promoted back to Member
var expectedVal = (MembersCount : 3, RehabsCount : 0);
var val = await WaitForPredicateAsync(async() => await RavenDB_15588.GetMembersAndRehabsCount(store), expectedVal : expectedVal);
Assert.True(val.Equals(expectedVal), await GetDecisionsForDatabase(
new StringBuilder("Failed on asserting Members and Rehabs count.")
.AppendLine($"Expected {expectedVal}, got : {val}")));
}
}
public async Task Stats()
{
var name = GetStringQueryString("name", required: false);
using (var context = QueryOperationContext.Allocate(Database, needsServerContext: true))
await using (var writer = new AsyncBlittableJsonTextWriter(context.Documents, ResponseBodyStream()))
{
IndexStats[] indexStats;
using (context.OpenReadTransaction())
{
if (string.IsNullOrEmpty(name))
{
indexStats = Database.IndexStore
.GetIndexes()
.OrderBy(x => x.Name)
.Select(x =>
{
try
{
return(x.GetStats(calculateLag: true, calculateStaleness: true, calculateMemoryStats: true, queryContext: context));
}
catch (Exception e)
{
if (Logger.IsOperationsEnabled)
{
Logger.Operations($"Failed to get stats of '{x.Name}' index", e);
}
try
{
Database.NotificationCenter.Add(AlertRaised.Create(Database.Name, $"Failed to get stats of '{x.Name}' index",
$"Exception was thrown on getting stats of '{x.Name}' index",
AlertType.Indexing_CouldNotGetStats, NotificationSeverity.Error, key: x.Name, details: new ExceptionDetails(e)));
}
catch (Exception addAlertException)
{
if (Logger.IsOperationsEnabled && addAlertException.IsOutOfMemory() == false && addAlertException.IsRavenDiskFullException() == false)
{
Logger.Operations($"Failed to add alert when getting error on retrieving stats of '{x.Name}' index", addAlertException);
}
}
var state = x.State;
if (e.IsOutOfMemory() == false && e.IsRavenDiskFullException() == false)
{
try
{
state = IndexState.Error;
x.SetState(state, inMemoryOnly: true);
}
catch (Exception ex)
{
if (Logger.IsOperationsEnabled)
{
Logger.Operations($"Failed to change state of '{x.Name}' index to error after encountering exception when getting its stats.",
ex);
}
}
}
return(new IndexStats
{
Name = x.Name,
Type = x.Type,
State = state,
Status = x.Status,
LockMode = x.Definition.LockMode,
Priority = x.Definition.Priority,
});
}
})
.ToArray();
}
else
{
var index = Database.IndexStore.GetIndex(name);
if (index == null)
{
HttpContext.Response.StatusCode = (int)HttpStatusCode.NotFound;
return;
}
indexStats = new[] { index.GetStats(calculateLag: true, calculateStaleness: true, calculateMemoryStats: true, queryContext: context) };
}
}
writer.WriteStartObject();
writer.WriteArray(context.Documents, "Results", indexStats, (w, c, stats) => w.WriteIndexStats(context.Documents, stats));
writer.WriteEndObject();
}
}
public async Task StreamQueryGet()
{
// ReSharper disable once ArgumentsStyleLiteral
using (var tracker = new RequestTimeTracker(HttpContext, Logger, Database, "StreamQuery", doPerformanceHintIfTooLong: false))
using (var token = CreateTimeLimitedQueryToken())
using (var queryContext = QueryOperationContext.Allocate(Database))
{
var documentId = GetStringQueryString("fromDocument", false);
string overrideQuery = null;
if (string.IsNullOrEmpty(documentId) == false)
{
Document document;
using (queryContext.OpenReadTransaction())
{
document = Database.DocumentsStorage.Get(queryContext.Documents, documentId);
if (document == null)
{
throw new DocumentDoesNotExistException($"Was request to stream a query taken from {documentId} document, but it does not exist.");
}
if (document.Data.TryGet("Query", out overrideQuery) == false)
{
throw new MissingFieldException($"Expected {documentId} to have a property named 'Query' of type 'String' but couldn't locate such property.");
}
}
}
var query = IndexQueryServerSide.Create(HttpContext, GetStart(), GetPageSize(), queryContext.Documents, tracker, overrideQuery);
var format = GetStringQueryString("format", false);
var debug = GetStringQueryString("debug", false);
var properties = GetStringValuesQueryString("field", false);
var propertiesArray = properties.Count == 0 ? null : properties.ToArray();
// set the exported file name prefix
var fileNamePrefix = query.Metadata.IsCollectionQuery ? query.Metadata.CollectionName + "_collection" : "query_result";
fileNamePrefix = $"{Database.Name}_{fileNamePrefix}";
if (string.IsNullOrWhiteSpace(debug) == false)
{
if (string.Equals(debug, "entries", StringComparison.OrdinalIgnoreCase))
{
using (var writer = GetIndexEntriesQueryResultWriter(format, HttpContext.Response, ResponseBodyStream(), propertiesArray, fileNamePrefix))
{
try
{
await Database.QueryRunner.ExecuteStreamIndexEntriesQuery(query, queryContext, HttpContext.Response, writer, token).ConfigureAwait(false);
}
catch (IndexDoesNotExistException)
{
HttpContext.Response.StatusCode = (int)HttpStatusCode.NotFound;
writer.WriteError($"Index {query.Metadata.IndexName} does not exist");
}
}
}
else
{
ThrowUnsupportedException($"You have selected {debug} debug mode, which is not supported.");
}
}
else
{
using (var writer = GetQueryResultWriter(format, HttpContext.Response, queryContext.Documents, ResponseBodyStream(), propertiesArray, fileNamePrefix))
{
try
{
await Database.QueryRunner.ExecuteStreamQuery(query, queryContext, HttpContext.Response, writer, token).ConfigureAwait(false);
}
catch (IndexDoesNotExistException)
{
HttpContext.Response.StatusCode = (int)HttpStatusCode.NotFound;
writer.WriteError($"Index {query.Metadata.IndexName} does not exist");
}
catch (Exception e)
{
try
{
writer.WriteError($"Failed to execute stream query. Error: {e}");
}
catch (Exception ie)
{
if (Logger.IsOperationsEnabled)
{
Logger.Operations($"Failed to write error. Error: {e}", ie);
}
}
throw;
}
}
}
}
}
public async Task StreamQueryPost()
{
// ReSharper disable once ArgumentsStyleLiteral
using (var tracker = new RequestTimeTracker(HttpContext, Logger, Database, "StreamQuery", doPerformanceHintIfTooLong: false))
using (var token = CreateTimeLimitedQueryToken())
using (var queryContext = QueryOperationContext.Allocate(Database))
{
var stream = TryGetRequestFromStream("ExportOptions") ?? RequestBodyStream();
var queryJson = await queryContext.Documents.ReadForMemoryAsync(stream, "index/query");
var query = IndexQueryServerSide.Create(HttpContext, queryJson, Database.QueryMetadataCache, tracker);
if (TrafficWatchManager.HasRegisteredClients)
{
var sb = new StringBuilder();
// append stringBuilder with the query
sb.Append(query.Query);
// if query got parameters append with parameters
if (query.QueryParameters != null && query.QueryParameters.Count > 0)
{
sb.AppendLine().Append(query.QueryParameters);
}
AddStringToHttpContext(sb.ToString(), TrafficWatchChangeType.Streams);
}
var format = GetStringQueryString("format", false);
var debug = GetStringQueryString("debug", false);
var properties = GetStringValuesQueryString("field", false);
var propertiesArray = properties.Count == 0 ? null : properties.ToArray();
// set the exported file name prefix
var fileNamePrefix = query.Metadata.IsCollectionQuery ? query.Metadata.CollectionName + "_collection" : "query_result";
fileNamePrefix = $"{Database.Name}_{fileNamePrefix}";
if (string.IsNullOrWhiteSpace(debug) == false)
{
if (string.Equals(debug, "entries", StringComparison.OrdinalIgnoreCase))
{
using (var writer = GetIndexEntriesQueryResultWriter(format, HttpContext.Response, ResponseBodyStream(), propertiesArray, fileNamePrefix))
{
try
{
await Database.QueryRunner.ExecuteStreamIndexEntriesQuery(query, queryContext, HttpContext.Response, writer, token).ConfigureAwait(false);
}
catch (IndexDoesNotExistException)
{
HttpContext.Response.StatusCode = (int)HttpStatusCode.NotFound;
writer.WriteError($"Index {query.Metadata.IndexName} does not exist");
}
}
}
else
{
ThrowUnsupportedException($"You have selected {debug} debug mode, which is not supported.");
}
}
else
{
using (var writer = GetQueryResultWriter(format, HttpContext.Response, queryContext.Documents, ResponseBodyStream(), propertiesArray, fileNamePrefix))
{
try
{
await Database.QueryRunner.ExecuteStreamQuery(query, queryContext, HttpContext.Response, writer, token).ConfigureAwait(false);
}
catch (IndexDoesNotExistException)
{
HttpContext.Response.StatusCode = (int)HttpStatusCode.NotFound;
writer.WriteError($"Index {query.Metadata.IndexName} does not exist");
}
}
}
}
}
private Dictionary <string, DatabaseStatusReport> CollectDatabaseInformation(TransactionOperationContext ctx, Dictionary <string, DatabaseStatusReport> prevReport)
{
var result = new Dictionary <string, DatabaseStatusReport>();
foreach (var dbName in _server.Cluster.GetDatabaseNames(ctx))
{
if (_token.IsCancellationRequested)
{
return(result);
}
var report = new DatabaseStatusReport
{
Name = dbName,
NodeName = _server.NodeTag
};
if (_server.DatabasesLandlord.DatabasesCache.TryGetValue(dbName, out var dbTask, out var details) == false)
{
DatabaseTopology topology;
using (var rawRecord = _server.Cluster.ReadRawDatabaseRecord(ctx, dbName))
{
if (rawRecord == null)
{
continue; // Database does not exists in this server
}
topology = rawRecord.Topology;
}
if (topology == null)
{
continue;
}
if (topology.RelevantFor(_server.NodeTag) == false)
{
continue;
}
report.Status = DatabaseStatus.Unloaded;
result[dbName] = report;
continue;
}
report.UpTime = SystemTime.UtcNow - details.InCacheSince;
if (dbTask.IsFaulted)
{
var extractSingleInnerException = dbTask.Exception.ExtractSingleInnerException();
if (Equals(extractSingleInnerException.Data[DatabasesLandlord.DoNotRemove], true))
{
report.Status = DatabaseStatus.Unloaded;
result[dbName] = report;
continue;
}
}
if (dbTask.IsCanceled || dbTask.IsFaulted)
{
report.Status = DatabaseStatus.Faulted;
report.Error = dbTask.Exception.ToString();
result[dbName] = report;
continue;
}
if (dbTask.IsCompleted == false)
{
report.Status = DatabaseStatus.Loading;
result[dbName] = report;
continue;
}
var dbInstance = dbTask.Result;
var currentHash = dbInstance.GetEnvironmentsHash();
report.EnvironmentsHash = currentHash;
var documentsStorage = dbInstance.DocumentsStorage;
var indexStorage = dbInstance.IndexStore;
if (dbInstance.DatabaseShutdown.IsCancellationRequested)
{
report.Status = DatabaseStatus.Shutdown;
result[dbName] = report;
continue;
}
report.Status = DatabaseStatus.Loaded;
try
{
var now = dbInstance.Time.GetUtcNow();
FillReplicationInfo(dbInstance, report);
prevReport.TryGetValue(dbName, out var prevDatabaseReport);
if (SupportedFeatures.Heartbeats.SendChangesOnly &&
prevDatabaseReport != null && prevDatabaseReport.EnvironmentsHash == currentHash)
{
report.Status = DatabaseStatus.NoChange;
result[dbName] = report;
continue;
}
using (var context = QueryOperationContext.Allocate(dbInstance, needsServerContext: true))
{
FillDocumentsInfo(prevDatabaseReport, dbInstance, report, context.Documents, documentsStorage);
FillClusterTransactionInfo(report, dbInstance);
if (indexStorage != null)
{
foreach (var index in indexStorage.GetIndexes())
{
DatabaseStatusReport.ObservedIndexStatus stat = null;
if (prevDatabaseReport?.LastIndexStats.TryGetValue(index.Name, out stat) == true && stat?.LastTransactionId == index.LastTransactionId)
{
report.LastIndexStats[index.Name] = stat;
continue;
}
using (context.OpenReadTransaction())
{
FillIndexInfo(index, context, now, report);
}
}
}
}
}
catch (Exception e)
{
report.EnvironmentsHash = 0; // on error we should do the complete report collaction path
report.Error = e.ToString();
}
result[dbName] = report;
}
return(result);
}
