private void FillDatabaseStatistics(DatabaseStatistics stats, QueryOperationContext context)
{
var indexes = Database.IndexStore.GetIndexes().ToList();
var size = Database.GetSizeOnDisk();
stats.LastDocEtag = DocumentsStorage.ReadLastDocumentEtag(context.Documents.Transaction.InnerTransaction);
stats.LastDatabaseEtag = DocumentsStorage.ReadLastEtag(context.Documents.Transaction.InnerTransaction);
stats.DatabaseChangeVector = DocumentsStorage.GetDatabaseChangeVector(context.Documents);
stats.CountOfDocuments = Database.DocumentsStorage.GetNumberOfDocuments(context.Documents);
stats.CountOfRevisionDocuments = Database.DocumentsStorage.RevisionsStorage.GetNumberOfRevisionDocuments(context.Documents);
stats.CountOfDocumentsConflicts = Database.DocumentsStorage.ConflictsStorage.GetNumberOfDocumentsConflicts(context.Documents);
stats.CountOfTombstones = Database.DocumentsStorage.GetNumberOfTombstones(context.Documents);
stats.CountOfConflicts = Database.DocumentsStorage.ConflictsStorage.ConflictsCount;
stats.SizeOnDisk = size.Data;
stats.NumberOfTransactionMergerQueueOperations = Database.TxMerger.NumberOfQueuedOperations;
stats.TempBuffersSizeOnDisk = size.TempBuffers;
stats.CountOfCounterEntries = Database.DocumentsStorage.CountersStorage.GetNumberOfCounterEntries(context.Documents);
stats.CountOfTimeSeriesSegments = Database.DocumentsStorage.TimeSeriesStorage.GetNumberOfTimeSeriesSegments(context.Documents);
var attachments = Database.DocumentsStorage.AttachmentsStorage.GetNumberOfAttachments(context.Documents);
stats.CountOfAttachments = attachments.AttachmentCount;
stats.CountOfUniqueAttachments = attachments.StreamsCount;
stats.CountOfIndexes = indexes.Count;
stats.DatabaseId = Database.DocumentsStorage.Environment.Base64Id;
stats.Is64Bit = !Database.DocumentsStorage.Environment.Options.ForceUsing32BitsPager && IntPtr.Size == sizeof(long);
stats.Pager = Database.DocumentsStorage.Environment.Options.DataPager.GetType().ToString();
stats.Indexes = new IndexInformation[indexes.Count];
for (var i = 0; i < indexes.Count; i++)
{
var index = indexes[i];
bool isStale;
try
{
isStale = index.IsStale(context);
}
catch (OperationCanceledException)
{
// if the index has just been removed, let us consider it stale
// until it can be safely removed from the list of indexes in the
// database
isStale = true;
}
stats.Indexes[i] = new IndexInformation
{
State = index.State,
IsStale = isStale,
Name = index.Name,
LockMode = index.Definition.LockMode,
Priority = index.Definition.Priority,
Type = index.Type,
LastIndexingTime = index.LastIndexingTime,
SourceType = index.SourceType
};
if (stats.LastIndexingTime.HasValue)
{
stats.LastIndexingTime = stats.LastIndexingTime >= index.LastIndexingTime ? stats.LastIndexingTime : index.LastIndexingTime;
}
else
{
stats.LastIndexingTime = index.LastIndexingTime;
}
}
}
/// <summary>
/// Setup a RavenDb database, ready to be used with some seeded, fake data (if provided) and any indexes (if provided).
/// </summary>
/// <param name="documentStore">DocumentStore to check.</param>
/// <param name="setupOptions">Optional: any custom setup options, like data to seed.</param>
/// <param name="logger">Logger.</param>
/// <param name="cancellationToken">Cancellation token.</param>
/// <remarks>Default Polly policy is 15 checks, every 2 seconds (if none was provided).</remarks>
public static async Task SetupRavenDbAsync(this IDocumentStore documentStore,
RavenDbSetupOptions setupOptions,
ILogger logger,
CancellationToken cancellationToken)
{
if (documentStore == null)
{
throw new ArgumentNullException(nameof(documentStore));
}
if (logger == null)
{
throw new ArgumentNullException(nameof(logger));
}
logger.LogDebug($" - Checking if the database '{documentStore.Database}' exists and if any data exists...");
DatabaseStatistics existingDatabaseStatistics = null;
try
{
var checkRavenDbPolicy = setupOptions?.Policy ?? CheckRavenDbPolicyAsync(logger, cancellationToken);
await checkRavenDbPolicy.ExecuteAsync(async cancellationToken =>
{
existingDatabaseStatistics = await documentStore.Maintenance.SendAsync(new GetStatisticsOperation(), cancellationToken);
}, cancellationToken);
}
catch (DatabaseDoesNotExistException)
{
existingDatabaseStatistics = null; // No statistics because there's no Db tenant.
}
catch (AuthorizationException)
{
// We failed to authenticate against the database. This _usually_ means that we
// probably didn't have ADMIN rights against a db (so we can't do high level stuff)
// but we could still do other stuff, like add fake data for seeding the DB.
// SUMMARY: Db Tenant might exist, so lets assume that it does (safer bet).
existingDatabaseStatistics = new DatabaseStatistics();
}
if (cancellationToken.IsCancellationRequested)
{
logger.LogInformation("Cancelling setting up RavenDb database ...");
cancellationToken.ThrowIfCancellationRequested();
}
try
{
await SetupRavenDbPolicyAsync(logger, cancellationToken).ExecuteAsync(async cancellationToken =>
{
await SetupDatabaseTenantAsync(documentStore,
existingDatabaseStatistics != null,
logger,
cancellationToken);
if (cancellationToken.IsCancellationRequested)
{
logger.LogInformation("Cancelling setting up RavenDb database ...");
cancellationToken.ThrowIfCancellationRequested();
}
var documentCount = existingDatabaseStatistics?.CountOfDocuments ?? 0;
if (documentCount > 0)
{
logger.LogDebug($" - Skipping seeding fake data because database has {documentCount:N0} documents already in it.");
}
else
{
await SeedCollectionsOfFakeDataAsync(documentStore,
setupOptions?.DocumentCollections,
logger,
cancellationToken);
}
if (cancellationToken.IsCancellationRequested)
{
logger.LogInformation("Cancelling setting up RavenDb database ...");
cancellationToken.ThrowIfCancellationRequested();
}
await SetupIndexesAsync(documentStore,
setupOptions?.IndexAssembly,
logger,
cancellationToken);
}, cancellationToken);
}
catch (Exception exception)
{
logger.LogError("Failed to setup RavenDb and all retries failed. Unable to continue. Application is now terminating. Error: {exception}", exception);
throw;
}
}
public static void WriteDatabaseStatistics(this BlittableJsonTextWriter writer, JsonOperationContext context, DatabaseStatistics statistics)
{
writer.WriteStartObject();
writer.WritePropertyName(nameof(statistics.CountOfIndexes));
writer.WriteInteger(statistics.CountOfIndexes);
writer.WriteComma();
writer.WritePropertyName(nameof(statistics.CountOfDocuments));
writer.WriteInteger(statistics.CountOfDocuments);
writer.WriteComma();
if (statistics.CountOfRevisionDocuments > 0)
{
writer.WritePropertyName(nameof(statistics.CountOfRevisionDocuments));
writer.WriteInteger(statistics.CountOfRevisionDocuments);
writer.WriteComma();
}
writer.WritePropertyName(nameof(statistics.CountOfAttachments));
writer.WriteInteger(statistics.CountOfAttachments);
writer.WriteComma();
writer.WritePropertyName(nameof(statistics.CountOfUniqueAttachments));
writer.WriteInteger(statistics.CountOfUniqueAttachments);
writer.WriteComma();
writer.WritePropertyName(nameof(statistics.DatabaseChangeVector));
writer.WriteString(statistics.DatabaseChangeVector);
writer.WriteComma();
writer.WritePropertyName(nameof(statistics.DatabaseId));
writer.WriteString(statistics.DatabaseId);
writer.WriteComma();
writer.WritePropertyName(nameof(statistics.Is64Bit));
writer.WriteBool(statistics.Is64Bit);
writer.WriteComma();
writer.WritePropertyName(nameof(statistics.Pager));
writer.WriteString(statistics.Pager);
writer.WriteComma();
writer.WritePropertyName(nameof(statistics.LastDocEtag));
if (statistics.LastDocEtag.HasValue)
{
writer.WriteInteger(statistics.LastDocEtag.Value);
}
else
{
writer.WriteNull();
}
writer.WriteComma();
writer.WritePropertyName((nameof(statistics.DatabaseChangeVector)));
writer.WriteString(statistics.DatabaseChangeVector);
writer.WriteComma();
writer.WritePropertyName(nameof(statistics.LastIndexingTime));
if (statistics.LastIndexingTime.HasValue)
{
writer.WriteDateTime(statistics.LastIndexingTime.Value, isUtc: true);
}
else
{
writer.WriteNull();
}
writer.WriteComma();
writer.WritePropertyName(nameof(statistics.Indexes));
writer.WriteStartArray();
var isFirstInternal = true;
foreach (var index in statistics.Indexes)
{
if (isFirstInternal == false)
{
writer.WriteComma();
}
isFirstInternal = false;
writer.WriteStartObject();
writer.WritePropertyName(nameof(index.IsStale));
writer.WriteBool(index.IsStale);
writer.WriteComma();
writer.WritePropertyName(nameof(index.Name));
writer.WriteString(index.Name);
writer.WriteComma();
writer.WritePropertyName(nameof(index.Etag));
writer.WriteInteger(index.Etag);
writer.WriteComma();
writer.WritePropertyName(nameof(index.LockMode));
writer.WriteString(index.LockMode.ToString());
writer.WriteComma();
writer.WritePropertyName(nameof(index.Priority));
writer.WriteString(index.Priority.ToString());
writer.WriteComma();
writer.WritePropertyName(nameof(index.State));
writer.WriteString(index.State.ToString());
writer.WriteComma();
writer.WritePropertyName(nameof(index.Type));
writer.WriteString(index.Type.ToString());
writer.WriteComma();
writer.WritePropertyName(nameof(index.LastIndexingTime));
if (index.LastIndexingTime.HasValue)
{
writer.WriteDateTime(index.LastIndexingTime.Value, isUtc: true);
}
else
{
writer.WriteNull();
}
writer.WriteEndObject();
}
writer.WriteEndArray();
writer.WriteEndObject();
}
public Task Stats()
{
using (ContextPool.AllocateOperationContext(out DocumentsOperationContext context))
using (var writer = new BlittableJsonTextWriter(context, ResponseBodyStream()))
using (context.OpenReadTransaction())
{
var indexes = Database.IndexStore.GetIndexes().ToList();
var sizeOnDiskInBytes = Database.GetSizeOnDiskInBytes();
var stats = new DatabaseStatistics
{
LastDocEtag = DocumentsStorage.ReadLastDocumentEtag(context.Transaction.InnerTransaction),
CountOfDocuments = Database.DocumentsStorage.GetNumberOfDocuments(context),
CountOfRevisionDocuments = Database.DocumentsStorage.RevisionsStorage.GetNumberOfRevisionDocuments(context),
CountOfDocumentsConflicts = Database.DocumentsStorage.ConflictsStorage.GetCountOfDocumentsConflicts(context),
CountOfConflicts = Database.DocumentsStorage.ConflictsStorage.ConflictsCount,
SizeOnDisk = new Size(sizeOnDiskInBytes)
};
var attachments = Database.DocumentsStorage.AttachmentsStorage.GetNumberOfAttachments(context);
stats.CountOfAttachments = attachments.AttachmentCount;
stats.CountOfUniqueAttachments = attachments.StreamsCount;
stats.CountOfIndexes = indexes.Count;
var statsDatabaseChangeVector = DocumentsStorage.GetDatabaseChangeVector(context);
stats.DatabaseChangeVector = statsDatabaseChangeVector;
stats.DatabaseId = Database.DocumentsStorage.Environment.Base64Id;
stats.Is64Bit = !Database.DocumentsStorage.Environment.Options.ForceUsing32BitsPager && IntPtr.Size == sizeof(long);
stats.Pager = Database.DocumentsStorage.Environment.Options.DataPager.GetType().ToString();
stats.Indexes = new IndexInformation[indexes.Count];
for (var i = 0; i < indexes.Count; i++)
{
var index = indexes[i];
bool isStale;
try
{
isStale = index.IsStale(context);
}
catch (OperationCanceledException)
{
// if the index has just been removed, let us consider it stale
// until it can be safely removed from the list of indexes in the
// database
isStale = true;
}
stats.Indexes[i] = new IndexInformation
{
State = index.State,
IsStale = isStale,
Name = index.Name,
Etag = index.Etag,
LockMode = index.Definition.LockMode,
Priority = index.Definition.Priority,
Type = index.Type,
LastIndexingTime = index.LastIndexingTime
};
if (stats.LastIndexingTime.HasValue)
{
stats.LastIndexingTime = stats.LastIndexingTime >= index.LastIndexingTime ? stats.LastIndexingTime : index.LastIndexingTime;
}
else
{
stats.LastIndexingTime = index.LastIndexingTime;
}
}
writer.WriteDatabaseStatistics(context, stats);
}
return(Task.CompletedTask);
}
public async Task CanExportFromCurrentAndImportTo42(ExcludeOn excludeOn)
{
var file = GetTempFileName();
using var store42 = await GetDocumentStoreAsync(Server42Version);
using var storeCurrent = GetDocumentStore();
//Export
storeCurrent.Maintenance.Send(new CreateSampleDataOperation());
using (var session = storeCurrent.OpenAsyncSession())
{
var dateTime = new DateTime(2020, 3, 29);
for (var i = 0; i < 5; i++)
{
var user = new User {
Name = "raven" + i
};
await session.StoreAsync(user);
session.TimeSeriesFor(user, "Heartrate").Append(dateTime, 59d, "watches/fitbit");
}
await session.SaveChangesAsync();
}
var exportOptions = new DatabaseSmugglerExportOptions();
if (excludeOn == ExcludeOn.Export)
{
exportOptions.OperateOnTypes &= ~(DatabaseItemType.Attachments | DatabaseItemType.RevisionDocuments | DatabaseItemType.CounterGroups);
}
var exportOperation = await storeCurrent.Smuggler.ExportAsync(exportOptions, file);
await exportOperation.WaitForCompletionAsync(_operationTimeout);
DatabaseStatistics expected = await storeCurrent.Maintenance.SendAsync(new GetStatisticsOperation());
//Import
var importOptions = new DatabaseSmugglerImportOptions {
SkipRevisionCreation = true
};
importOptions.OperateOnTypes &= ~DatabaseItemType.TimeSeries;
if (excludeOn == ExcludeOn.Import)
{
importOptions.OperateOnTypes &= ~(DatabaseItemType.Attachments | DatabaseItemType.RevisionDocuments | DatabaseItemType.CounterGroups);
}
var importOperation = await store42.Smuggler.ImportAsync(importOptions, file);
await importOperation.WaitForCompletionAsync(_operationTimeout);
var actual = await store42.Maintenance.SendAsync(new GetStatisticsOperation());
//Assert
Assert.Equal(expected.CountOfIndexes, actual.CountOfIndexes);
Assert.Equal(expected.CountOfDocuments, actual.CountOfDocuments);
var export = await GetMetadataCounts(storeCurrent);
var import = await GetMetadataCounts(store42);
if (excludeOn == ExcludeOn.Non)
{
Assert.Equal(expected.CountOfAttachments, actual.CountOfAttachments);
Assert.Equal(expected.CountOfRevisionDocuments, actual.CountOfRevisionDocuments);
Assert.Equal(expected.CountOfCounterEntries, actual.CountOfCounterEntries);
Assert.Equal(export, import);
}
else
{
Assert.Equal(0, actual.CountOfAttachments);
Assert.Equal(0, actual.CountOfRevisionDocuments);
Assert.Equal(0, actual.CountOfCounterEntries);
Assert.Equal((0, 0, 0), import);
}
}
public StatisticsUpdated(DatabaseStatistics statistics) { Statistics = statistics; }
private static bool AssertReplication(DocumentStore destinationStore, string database, DatabaseStatistics sourceStats)
{
var destinationStats = destinationStore.Maintenance.ForDatabase(database).Send(new GetStatisticsOperation());
if (destinationStats.CountOfAttachments == sourceStats.CountOfAttachments &&
destinationStats.CountOfDocuments == sourceStats.CountOfDocuments &&
destinationStats.CountOfRevisionDocuments == sourceStats.CountOfRevisionDocuments &&
destinationStats.CountOfUniqueAttachments == sourceStats.CountOfUniqueAttachments)
{
return(true);
}
return(false);
}
public StatisticsUpdated(DatabaseStatistics statistics)
{
Statistics = statistics;
}
