public async Task CanRecoverEncryptedDatabase()
{
string dbName = SetupEncryptedDatabase(out X509Certificate2 adminCert, out var masterKey);
var dbPath = NewDataPath(prefix: Guid.NewGuid().ToString());
var recoveryExportPath = NewDataPath(prefix: Guid.NewGuid().ToString());
DatabaseStatistics databaseStatistics;
using (var store = GetDocumentStore(new Options()
{
AdminCertificate = adminCert,
ClientCertificate = adminCert,
ModifyDatabaseName = s => dbName,
ModifyDatabaseRecord = record => record.Encrypted = true,
Path = dbPath
}))
{
store.Maintenance.Send(new CreateSampleDataOperation());
databaseStatistics = store.Maintenance.Send(new GetStatisticsOperation());
}
using (var recovery = new Recovery(new VoronRecoveryConfiguration()
{
LoggingMode = Sparrow.Logging.LogMode.None,
DataFileDirectory = dbPath,
PathToDataFile = Path.Combine(dbPath, "Raven.voron"),
OutputFileName = Path.Combine(recoveryExportPath, "recovery.ravendump"),
MasterKey = masterKey
}))
{
recovery.Execute(TextWriter.Null, CancellationToken.None);
}
using (var store = GetDocumentStore(new Options()
{
AdminCertificate = adminCert,
ClientCertificate = adminCert,
}))
{
var op = await store.Smuggler.ImportAsync(new DatabaseSmugglerImportOptions()
{
}, Path.Combine(recoveryExportPath, "recovery-2-Documents.ravendump"));
op.WaitForCompletion(TimeSpan.FromMinutes(2));
var currentStats = store.Maintenance.Send(new GetStatisticsOperation());
// + 1 as recovery adds some artificial items
Assert.Equal(databaseStatistics.CountOfAttachments + 1, currentStats.CountOfAttachments);
Assert.Equal(databaseStatistics.CountOfDocuments + 1, currentStats.CountOfDocuments);
}
}
public async Task <DocumentStore> RecoverDatabase(RecoveryOptions options)
{
var recoveryExportPath = options.RecoveryDirectory;
using (var recovery = new Recovery(new VoronRecoveryConfiguration
{
LoggingMode = Sparrow.Logging.LogMode.None,
DataFileDirectory = options.PathToDataFile,
PathToDataFile = Path.Combine(options.PathToDataFile, "Raven.voron"),
OutputFileName = Path.Combine(recoveryExportPath, "recovery.ravendbdump"),
MasterKey = options.MasterKey,
DisableCopyOnWriteMode = options.DisableCopyOnWriteMode
}))
{
recovery.Execute(Console.Out, CancellationToken.None);
}
var store = GetDocumentStore(new Options {
AdminCertificate = options.AdminCertificate, ClientCertificate = options.ClientCertificate
});
if (options.RecoveryTypes.HasFlag(RecoveryTypes.Documents))
{
await ImportFile(store, recoveryExportPath, "recovery-2-Documents.ravendbdump");
}
if (options.RecoveryTypes.HasFlag(RecoveryTypes.Revisions))
{
await store.Maintenance.SendAsync(new ConfigureRevisionsOperation(new RevisionsConfiguration {
Default = new RevisionsCollectionConfiguration {
}
}));
await ImportFile(store, recoveryExportPath, "recovery-3-Revisions.ravendbdump");
}
if (options.RecoveryTypes.HasFlag(RecoveryTypes.Conflicts))
{
await ImportFile(store, recoveryExportPath, "recovery-4-Conflicts.ravendbdump");
}
if (options.RecoveryTypes.HasFlag(RecoveryTypes.Counters))
{
await ImportFile(store, recoveryExportPath, "recovery-5-Counters.ravendbdump");
}
if (options.RecoveryTypes.HasFlag(RecoveryTypes.TimeSeries))
{
await ImportFile(store, recoveryExportPath, "recovery-6-TimeSeries.ravendbdump");
}
return(store);
}
public async Task CanUseVoronRecoveryOnEmptyDatabase()
{
var dbPath = NewDataPath();
var recoveryExportPath = NewDataPath();
using (GetDocumentStore(new Options()
{
Path = dbPath
}))
{
}
using (var recovery = new Recovery(new VoronRecoveryConfiguration()
{
LoggingMode = Sparrow.Logging.LogMode.None,
DataFileDirectory = dbPath,
PathToDataFile = Path.Combine(dbPath, "Raven.voron"),
OutputFileName = Path.Combine(recoveryExportPath, "recovery.ravendump"),
}))
{
recovery.Execute(TextWriter.Null, CancellationToken.None);
}
using (var store = GetDocumentStore())
{
var op = await store.Smuggler.ImportAsync(new DatabaseSmugglerImportOptions()
{
}, Path.Combine(recoveryExportPath, "recovery-2-Documents.ravendump"));
op.WaitForCompletion(TimeSpan.FromMinutes(2));
var databaseStatistics = store.Maintenance.Send(new GetStatisticsOperation());
Assert.Equal(0, databaseStatistics.CountOfAttachments);
Assert.Equal(0, databaseStatistics.CountOfDocuments);
}
}
public async Task CanRecoverEncryptedDatabaseInternal(bool nullifyMasterKey = false)
{
string dbName = SetupEncryptedDatabase(out var certificates, out var masterKey);
if (nullifyMasterKey)
{
masterKey = null;
}
var dbPath = NewDataPath(prefix: Guid.NewGuid().ToString());
var recoveryExportPath = NewDataPath(prefix: Guid.NewGuid().ToString());
DatabaseStatistics databaseStatistics;
using (var store = GetDocumentStore(new Options()
{
AdminCertificate = certificates.ServerCertificate.Value,
ClientCertificate = certificates.ServerCertificate.Value,
ModifyDatabaseName = s => dbName,
ModifyDatabaseRecord = record => record.Encrypted = true,
Path = dbPath
}))
{
store.Maintenance.Send(new CreateSampleDataOperation());
databaseStatistics = store.Maintenance.Send(new GetStatisticsOperation());
}
using (var recovery = new Recovery(new VoronRecoveryConfiguration()
{
LoggingMode = Sparrow.Logging.LogMode.None,
DataFileDirectory = dbPath,
PathToDataFile = Path.Combine(dbPath, "Raven.voron"),
OutputFileName = Path.Combine(recoveryExportPath, "recovery.ravendump"),
MasterKey = masterKey,
DisableCopyOnWriteMode = nullifyMasterKey
}))
{
recovery.Execute(TextWriter.Null, CancellationToken.None);
}
using (var store = GetDocumentStore(new Options()
{
AdminCertificate = certificates.ServerCertificate.Value,
ClientCertificate = certificates.ServerCertificate.Value,
}))
{
var op = await store.Smuggler.ImportAsync(new DatabaseSmugglerImportOptions()
{
}, Path.Combine(recoveryExportPath, "recovery-2-Documents.ravendump"));
op.WaitForCompletion(TimeSpan.FromMinutes(2));
var currentStats = store.Maintenance.Send(new GetStatisticsOperation());
// Temporary dump: RavenDB-13765
var msg = new StringBuilder();
if (databaseStatistics.CountOfAttachments + 1 != currentStats.CountOfAttachments ||
databaseStatistics.CountOfDocuments + 1 != currentStats.CountOfDocuments)
{
using (var session = store.OpenSession())
{
var list1 = session.Query <Employee>().ToList();
var list2 = session.Query <Category>().ToList();
foreach (Employee l1 in list1)
{
var opGetAttach = session.Advanced.Attachments.Get(l1, "photo.jpg");
var revCnt = session.Advanced.Revisions.GetFor <Employee>(l1.Id).Count;
msg.AppendLine($"Employee {l1.Id} Attachment = {opGetAttach?.Details?.DocumentId}, RevCount=" + revCnt);
}
foreach (Category l2 in list2)
{
var opGetAttach = session.Advanced.Attachments.Get(l2, "image.jpg");
var revCnt = session.Advanced.Revisions.GetFor <Employee>(l2.Id).Count;
msg.AppendLine($"Category {l2.Id} Attachment = {opGetAttach?.Details?.DocumentId}, RevCount=" + revCnt);
}
var documentDatabase = await GetDatabase(store.Database);
using (documentDatabase.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext context))
using (context.OpenReadTransaction())
{
var table = context.Transaction.InnerTransaction.OpenTable(AttachmentsStorage.AttachmentsSchema, AttachmentsStorage.AttachmentsMetadataSlice);
var count = table.NumberOfEntries;
var tree = context.Transaction.InnerTransaction.CreateTree(AttachmentsStorage.AttachmentsSlice);
var streamsCount = tree.State.NumberOfEntries;
msg.AppendLine($"count={count}, streamsCount={streamsCount}");
using (var it = tree.Iterate(false))
{
if (it.Seek(Slices.BeforeAllKeys))
{
do
{
msg.AppendLine("tree key=" + it.CurrentKey.Content);
} while (it.MoveNext());
}
}
}
}
var currentStats2 = store.Maintenance.Send(new GetStatisticsOperation());
msg.AppendLine("Get again currentStats.CountOfAttachments=" + currentStats2.CountOfAttachments);
throw new Exception(msg.ToString());
}
// + 1 as recovery adds some artificial items
Assert.Equal(databaseStatistics.CountOfAttachments + 1, currentStats.CountOfAttachments);
Assert.Equal(databaseStatistics.CountOfDocuments + 1, currentStats.CountOfDocuments);
}
}
public async Task CanLoadDatabaseAfterUsingVoronRecoveryOnItWithCopyOnWriteMode(bool compressed)
{
var dbPath = NewDataPath(prefix: Guid.NewGuid().ToString());
var recoveryExportPath = NewDataPath(prefix: Guid.NewGuid().ToString());
DatabaseStatistics databaseStatistics;
// create db with sample data
using (var store = GetDocumentStore(new Options()
{
Path = dbPath,
ModifyDatabaseRecord = record =>
{
if (compressed)
{
record.DocumentsCompression = new DocumentsCompressionConfiguration
{
Collections = new[] { "Orders", "Employees", "Companies", "Products" },
CompressRevisions = true
};
}
}
}))
{
await CreateLegacyNorthwindDatabase(store);
databaseStatistics = store.Maintenance.Send(new GetStatisticsOperation());
}
var journals = new DirectoryInfo(Path.Combine(dbPath, "Journals")).GetFiles();
// run recovery
using (var recovery = new Recovery(new VoronRecoveryConfiguration()
{
LoggingMode = Sparrow.Logging.LogMode.None,
DataFileDirectory = dbPath,
PathToDataFile = Path.Combine(dbPath, "Raven.voron"),
OutputFileName = Path.Combine(recoveryExportPath, "recovery.ravendump"),
}))
{
recovery.Execute(TextWriter.Null, CancellationToken.None);
}
// make sure no journal file was lost during the process - by default we use copy on write mode
var journalsAfterRecovery = new DirectoryInfo(Path.Combine(dbPath, "Journals")).GetFiles();
Assert.Equal(journals.Length, journalsAfterRecovery.Length);
Assert.True(journals.All(x => journalsAfterRecovery.Any(y => y.Name == x.Name)));
// let's open the database
using (var store = GetDocumentStore(new Options()
{
Path = dbPath
}))
{
var currentStats = store.Maintenance.Send(new GetStatisticsOperation());
Assert.Equal(databaseStatistics.CountOfAttachments, currentStats.CountOfAttachments);
Assert.Equal(databaseStatistics.CountOfDocuments, currentStats.CountOfDocuments);
}
// let's import the recovery files
using (var store = GetDocumentStore())
{
var op = await store.Smuggler.ImportAsync(new DatabaseSmugglerImportOptions()
{
}, Path.Combine(recoveryExportPath, "recovery-2-Documents.ravendump"));
op.WaitForCompletion(TimeSpan.FromMinutes(2));
var currentStats = store.Maintenance.Send(new GetStatisticsOperation());
// + 1 as recovery adds some artificial items
Assert.Equal(databaseStatistics.CountOfAttachments + 1, currentStats.CountOfAttachments);
Assert.Equal(databaseStatistics.CountOfDocuments + 1, currentStats.CountOfDocuments);
}
}
public async Task CanLoadDatabaseAfterUsingVoronRecoveryOnItWithCopyOnWriteMode()
{
var dbPath = NewDataPath(prefix: Guid.NewGuid().ToString());
var recoveryExportPath = NewDataPath(prefix: Guid.NewGuid().ToString());
DatabaseStatistics databaseStatistics;
// create db with sample data
using (var store = GetDocumentStore(new Options()
{
Path = dbPath
}))
{
store.Maintenance.Send(new CreateSampleDataOperation());
databaseStatistics = store.Maintenance.Send(new GetStatisticsOperation());
}
var journals = new DirectoryInfo(Path.Combine(dbPath, "Journals")).GetFiles();
// run recovery
using (var recovery = new Recovery(new VoronRecoveryConfiguration()
{
DataFileDirectory = dbPath,
PathToDataFile = Path.Combine(dbPath, "Raven.voron"),
OutputFileName = Path.Combine(recoveryExportPath, "recovery.ravendump"),
}))
{
recovery.Execute(CancellationToken.None);
}
// make sure no journal file was lost during the process - by default we use copy on write mode
var journalsAfterRecovery = new DirectoryInfo(Path.Combine(dbPath, "Journals")).GetFiles();
Assert.Equal(journals.Length, journalsAfterRecovery.Length);
Assert.True(journals.All(x => journalsAfterRecovery.Any(y => y.Name == x.Name)));
// let's open the database
using (var store = GetDocumentStore(new Options()
{
Path = dbPath
}))
{
var currentStats = store.Maintenance.Send(new GetStatisticsOperation());
Assert.Equal(databaseStatistics.CountOfAttachments, currentStats.CountOfAttachments);
Assert.Equal(databaseStatistics.CountOfDocuments, currentStats.CountOfDocuments);
}
// let's import the recovery files
using (var store = GetDocumentStore())
{
var op = await store.Smuggler.ImportAsync(new DatabaseSmugglerImportOptions()
{
}, Path.Combine(recoveryExportPath, "recovery-2-Documents.ravendump"));
op.WaitForCompletion(TimeSpan.FromMinutes(2));
var currentStats = store.Maintenance.Send(new GetStatisticsOperation());
// + 1 as recovery adds some artificial items
Assert.Equal(databaseStatistics.CountOfAttachments + 1, currentStats.CountOfAttachments);
Assert.Equal(databaseStatistics.CountOfDocuments + 1, currentStats.CountOfDocuments);
}
}