private async Task CanRecoverEncryptedDatabaseInternal(bool nullifyMasterKey = false, bool compressDocuments = false) { string dbName = Encryption.SetupEncryptedDatabase(out var certificates, out var masterKey); if (nullifyMasterKey) { masterKey = null; } var dbPath = NewDataPath(prefix: Guid.NewGuid().ToString()); var recoveryExportPath = NewDataPath(prefix: Guid.NewGuid().ToString()); DatabaseStatistics databaseStatistics; using (var store = GetDocumentStore(new Options() { AdminCertificate = certificates.ServerCertificate.Value, ClientCertificate = certificates.ServerCertificate.Value, ModifyDatabaseName = s => dbName, ModifyDatabaseRecord = record => { record.Encrypted = true; if (compressDocuments) { record.DocumentsCompression = new DocumentsCompressionConfiguration { Collections = new[] { "Orders", "Employees", "Companies", "Products" }, CompressRevisions = true }; } }, Path = dbPath })) { await Samples.CreateLegacyNorthwindDatabaseAsync(store); databaseStatistics = store.Maintenance.Send(new GetStatisticsOperation()); } using (var recovery = new Recovery(new VoronRecoveryConfiguration() { LoggingMode = Sparrow.Logging.LogMode.None, DataFileDirectory = dbPath, PathToDataFile = Path.Combine(dbPath, "Raven.voron"), OutputFileName = Path.Combine(recoveryExportPath, "recovery.ravendump"), MasterKey = masterKey, DisableCopyOnWriteMode = nullifyMasterKey })) { recovery.Execute(TextWriter.Null, CancellationToken.None); } using (var store = GetDocumentStore(new Options() { AdminCertificate = certificates.ServerCertificate.Value, ClientCertificate = certificates.ServerCertificate.Value, })) { var op = await store.Smuggler.ImportAsync(new DatabaseSmugglerImportOptions() { }, Path.Combine(recoveryExportPath, "recovery-2-Documents.ravendump")); op.WaitForCompletion(TimeSpan.FromMinutes(2)); var currentStats = store.Maintenance.Send(new GetStatisticsOperation()); // Temporary dump: RavenDB-13765 var msg = new StringBuilder(); if (databaseStatistics.CountOfAttachments + 1 != currentStats.CountOfAttachments || databaseStatistics.CountOfDocuments + 1 != currentStats.CountOfDocuments) { using (var session = store.OpenSession()) { var list1 = session.Query <Employee>().ToList(); var list2 = session.Query <Category>().ToList(); foreach (Employee l1 in list1) { using (var opGetAttach = session.Advanced.Attachments.Get(l1, "photo.jpg")) { var revCnt = session.Advanced.Revisions.GetFor <Employee>(l1.Id).Count; msg.AppendLine($"Employee {l1.Id} Attachment = {opGetAttach?.Details?.DocumentId}, RevCount=" + revCnt); } } foreach (Category l2 in list2) { using (var opGetAttach = session.Advanced.Attachments.Get(l2, "image.jpg")) { var revCnt = session.Advanced.Revisions.GetFor <Employee>(l2.Id).Count; msg.AppendLine($"Category {l2.Id} Attachment = {opGetAttach?.Details?.DocumentId}, RevCount=" + revCnt); } } var documentDatabase = await GetDatabase(store.Database); using (documentDatabase.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext context)) using (context.OpenReadTransaction()) { var table = context.Transaction.InnerTransaction.OpenTable(AttachmentsStorage.AttachmentsSchema, AttachmentsStorage.AttachmentsMetadataSlice); var count = table.NumberOfEntries; var tree = context.Transaction.InnerTransaction.CreateTree(AttachmentsStorage.AttachmentsSlice); var streamsCount = tree.State.NumberOfEntries; msg.AppendLine($"count={count}, streamsCount={streamsCount}"); using (var it = tree.Iterate(false)) { if (it.Seek(Slices.BeforeAllKeys)) { do { msg.AppendLine("tree key=" + it.CurrentKey.Content); } while (it.MoveNext()); } } } } var currentStats2 = store.Maintenance.Send(new GetStatisticsOperation()); msg.AppendLine("Get again currentStats.CountOfAttachments=" + currentStats2.CountOfAttachments); throw new Exception(msg.ToString()); } // + 1 as recovery adds some artificial items Assert.Equal(databaseStatistics.CountOfAttachments + 1, currentStats.CountOfAttachments); Assert.Equal(databaseStatistics.CountOfDocuments + 1, currentStats.CountOfDocuments); } }
public async Task CanLoadDatabaseAfterUsingVoronRecoveryOnItWithCopyOnWriteMode(bool compressed) { var dbPath = NewDataPath(prefix: Guid.NewGuid().ToString()); var recoveryExportPath = NewDataPath(prefix: Guid.NewGuid().ToString()); DatabaseStatistics databaseStatistics; // create db with sample data using (var store = GetDocumentStore(new Options() { Path = dbPath, ModifyDatabaseRecord = record => { if (compressed) { record.DocumentsCompression = new DocumentsCompressionConfiguration { Collections = new[] { "Orders", "Employees", "Companies", "Products" }, CompressRevisions = true }; } } })) { await Samples.CreateLegacyNorthwindDatabaseAsync(store); databaseStatistics = store.Maintenance.Send(new GetStatisticsOperation()); } var journals = new DirectoryInfo(Path.Combine(dbPath, "Journals")).GetFiles(); // run recovery using (var recovery = new Recovery(new VoronRecoveryConfiguration() { LoggingMode = Sparrow.Logging.LogMode.None, DataFileDirectory = dbPath, PathToDataFile = Path.Combine(dbPath, "Raven.voron"), OutputFileName = Path.Combine(recoveryExportPath, "recovery.ravendump"), })) { recovery.Execute(TextWriter.Null, CancellationToken.None); } // make sure no journal file was lost during the process - by default we use copy on write mode var journalsAfterRecovery = new DirectoryInfo(Path.Combine(dbPath, "Journals")).GetFiles(); Assert.Equal(journals.Length, journalsAfterRecovery.Length); Assert.True(journals.All(x => journalsAfterRecovery.Any(y => y.Name == x.Name))); // let's open the database using (var store = GetDocumentStore(new Options() { Path = dbPath })) { var currentStats = store.Maintenance.Send(new GetStatisticsOperation()); Assert.Equal(databaseStatistics.CountOfAttachments, currentStats.CountOfAttachments); Assert.Equal(databaseStatistics.CountOfDocuments, currentStats.CountOfDocuments); } // let's import the recovery files using (var store = GetDocumentStore()) { var op = await store.Smuggler.ImportAsync(new DatabaseSmugglerImportOptions() { }, Path.Combine(recoveryExportPath, "recovery-2-Documents.ravendump")); op.WaitForCompletion(TimeSpan.FromMinutes(2)); var currentStats = store.Maintenance.Send(new GetStatisticsOperation()); // + 1 as recovery adds some artificial items Assert.Equal(databaseStatistics.CountOfAttachments + 1, currentStats.CountOfAttachments); Assert.Equal(databaseStatistics.CountOfDocuments + 1, currentStats.CountOfDocuments); } }