public void AfterBackupRestoreCanQueryIndex_CreatedAfterRestore()
{
db.Documents.Put("ayende", null, RavenJObject.Parse("{'email':'*****@*****.**'}"), RavenJObject.Parse("{'Raven-Entity-Name':'Users'}"), null);
db.Maintenance.StartBackup(BackupDir, false, new DatabaseDocument());
WaitForBackup(db, true);
db.Dispose();
IOExtensions.DeleteDirectory(DataDir);
MaintenanceActions.Restore(new RavenConfiguration(), new DatabaseRestoreRequest
{
BackupLocation = BackupDir,
DatabaseLocation = DataDir
}, s => { });
db = new DocumentDatabase(new RavenConfiguration {
DataDirectory = DataDir
});
db.SpinBackgroundWorkers();
QueryResult queryResult;
do
{
queryResult = db.Queries.Query("Raven/DocumentsByEntityName", new IndexQuery
{
Query = "Tag:[[Users]]",
PageSize = 10
}, CancellationToken.None);
} while (queryResult.IsStale);
Assert.Equal(1, queryResult.Results.Count);
}
public void AfterBackupRestoreCanReadDocument()
{
db.Documents.Put("ayende", null, RavenJObject.Parse("{'email':'*****@*****.**'}"), new RavenJObject(), null);
db.Maintenance.StartBackup(BackupDir, false, new DatabaseDocument());
WaitForBackup(db, true);
db.Dispose();
IOExtensions.DeleteDirectory(DataDir);
MaintenanceActions.Restore(new RavenConfiguration(), new DatabaseRestoreRequest
{
BackupLocation = BackupDir,
DatabaseLocation = DataDir,
Defrag = true
}, s => { });
db = new DocumentDatabase(new RavenConfiguration {
DataDirectory = DataDir
});
var document = db.Documents.Get("ayende", null);
Assert.NotNull(document);
var jObject = document.ToJson();
Assert.Equal("*****@*****.**", jObject.Value <string>("email"));
}
private static void RunSystemDatabaseRestoreOperation(string backupLocation, string databaseLocation, bool defrag)
{
try
{
var ravenConfiguration = new RavenConfiguration();
if (File.Exists(Path.Combine(backupLocation, "Raven.voron")))
{
ravenConfiguration.DefaultStorageTypeName = typeof(Raven.Storage.Voron.TransactionalStorage).AssemblyQualifiedName;
}
else if (Directory.Exists(Path.Combine(backupLocation, "new")))
{
ravenConfiguration.DefaultStorageTypeName = typeof(Raven.Storage.Esent.TransactionalStorage).AssemblyQualifiedName;
}
MaintenanceActions.Restore(ravenConfiguration, new DatabaseRestoreRequest
{
BackupLocation = backupLocation,
DatabaseLocation = databaseLocation,
Defrag = defrag
}, Console.WriteLine);
}
catch (Exception e)
{
Console.WriteLine(e);
}
}
public void AfterFailedRestoreOfIndex_ShouldGenerateWarningAndResetIt(string storageName)
{
using (var db = new DocumentDatabase(new RavenConfiguration
{
DefaultStorageTypeName = storageName,
DataDirectory = DataDir,
RunInUnreliableYetFastModeThatIsNotSuitableForProduction = false,
}))
{
db.SpinBackgroundWorkers();
db.Indexes.PutIndex(new RavenDocumentsByEntityName().IndexName, new RavenDocumentsByEntityName().CreateIndexDefinition());
db.Documents.Put("users/1", null, RavenJObject.Parse("{'Name':'Arek'}"), RavenJObject.Parse("{'Raven-Entity-Name':'Users'}"), null);
db.Documents.Put("users/2", null, RavenJObject.Parse("{'Name':'David'}"), RavenJObject.Parse("{'Raven-Entity-Name':'Users'}"), null);
db.Documents.Put("users/3", null, RavenJObject.Parse("{'Name':'Daniel'}"), RavenJObject.Parse("{'Raven-Entity-Name':'Users'}"), null);
WaitForIndexing(db);
db.Maintenance.StartBackup(BackupDir, false, new DatabaseDocument());
WaitForBackup(db, true);
}
IOExtensions.DeleteDirectory(DataDir);
var path = Directory.GetFiles(BackupDir, "index-files.required-for-index-restore", SearchOption.AllDirectories).First();
// lock file to simulate IOException when restore operation will try to copy this file
using (var file = File.Open(path, FileMode.Open, FileAccess.ReadWrite, FileShare.None))
{
var sb = new StringBuilder();
MaintenanceActions.Restore(new RavenConfiguration(), new RestoreRequest
{
BackupLocation = BackupDir,
DatabaseLocation = DataDir
}, s => sb.Append(s));
Assert.Contains(
"could not be restored. All already copied index files was deleted." +
" Index will be recreated after launching Raven instance",
sb.ToString());
}
using (var db = new DocumentDatabase(new RavenConfiguration {
DataDirectory = DataDir
}))
{
db.SpinBackgroundWorkers();
QueryResult queryResult;
do
{
queryResult = db.Queries.Query("Raven/DocumentsByEntityName", new IndexQuery
{
Query = "Tag:[[Users]]",
PageSize = 10
}, CancellationToken.None);
} while (queryResult.IsStale);
Assert.Equal(3, queryResult.Results.Count);
}
}
public void AfterIncrementalBackupRestoreCanReadDocument(string storageName)
{
InitializeDocumentDatabase(storageName);
IOExtensions.DeleteDirectory(BackupDir);
db.Documents.Put("ayende", null, RavenJObject.Parse("{'email':'*****@*****.**'}"), new RavenJObject(), null);
db.Maintenance.StartBackup(BackupDir, false, new DatabaseDocument());
WaitForBackup(db, true);
db.Documents.Put("itamar", null, RavenJObject.Parse("{'email':'*****@*****.**'}"), new RavenJObject(), null);
db.Maintenance.StartBackup(BackupDir, true, new DatabaseDocument());
WaitForBackup(db, true);
db.Dispose();
IOExtensions.DeleteDirectory(DataDir);
MaintenanceActions.Restore(new RavenConfiguration
{
DefaultStorageTypeName = storageName,
DataDirectory = DataDir,
RunInMemory = false,
RunInUnreliableYetFastModeThatIsNotSuitableForProduction = false,
Settings =
{
{ "Raven/Esent/CircularLog", "false" },
{ "Raven/Voron/AllowIncrementalBackups", "true" }
}
}, new DatabaseRestoreRequest
{
BackupLocation = BackupDir,
DatabaseLocation = DataDir,
Defrag = true
}, s => { });
db = new DocumentDatabase(new RavenConfiguration {
DataDirectory = DataDir
});
var fetchedData = db.Documents.Get("ayende", null);
Assert.NotNull(fetchedData);
var jObject = fetchedData.ToJson();
Assert.NotNull(jObject);
Assert.Equal("*****@*****.**", jObject.Value <string>("email"));
fetchedData = db.Documents.Get("itamar", null);
Assert.NotNull(fetchedData);
jObject = fetchedData.ToJson();
Assert.NotNull(jObject);
Assert.Equal("*****@*****.**", jObject.Value <string>("email"));
}
public void NonIncrementalBackup_Restore_CanReadDocumentWithCorruptedIndex(string storageName)
{
InitializeDocumentDatabase(storageName);
IOExtensions.DeleteDirectory(BackupDir);
db.Documents.Put("Foo", null, RavenJObject.Parse("{'email':'*****@*****.**'}"), new RavenJObject(), null);
db.Maintenance.StartBackup(BackupDir, false, new DatabaseDocument(), new ResourceBackupState());
WaitForBackup(db, true);
db.Dispose();
IOExtensions.DeleteDirectory(DataDir);
var indexFile = Path.Combine(BackupDir, "IndexDefinitions", "1.index");
string text = File.ReadAllText(indexFile);
text = text.Replace("from", "corrupt");
File.WriteAllText(indexFile, text); // deliberately corrupt index
//index is corrupted --> should NOT fail to restore backup
Assert.DoesNotThrow(() =>
MaintenanceActions.Restore(new RavenConfiguration
{
DefaultStorageTypeName = storageName,
DataDirectory = DataDir,
RunInMemory = false,
RunInUnreliableYetFastModeThatIsNotSuitableForProduction = false,
Settings =
{
{ "Raven/Esent/CircularLog", "false" }
}
}, new DatabaseRestoreRequest
{
BackupLocation = BackupDir,
DatabaseLocation = DataDir,
Defrag = true
}, s => { }));
db = new DocumentDatabase(new RavenConfiguration {
DataDirectory = DataDir
}, null);
var fetchedData = db.Documents.Get("Foo", null);
Assert.NotNull(fetchedData);
var jObject = fetchedData.ToJson();
Assert.NotNull(jObject);
Assert.Equal("*****@*****.**", jObject.Value <string>("email"));
db.Dispose();
}
public void CannotRestoreFailedBackup(string storageName)
{
var config = new RavenConfiguration
{
DataDirectory = DataDir,
RunInUnreliableYetFastModeThatIsNotSuitableForProduction = false,
DefaultStorageTypeName = storageName,
Settings =
{
{ Constants.Esent.CircularLog, "false" },
{ Constants.Voron.AllowIncrementalBackups, "true" }
}
};
config.Storage.Voron.AllowIncrementalBackups = true;
using (db = new DocumentDatabase(config, null))
{
db.Indexes.PutIndex(new RavenDocumentsByEntityName().IndexName, new RavenDocumentsByEntityName().CreateIndexDefinition());
db.Documents.Put("ayende", null, RavenJObject.Parse("{'email':'*****@*****.**'}"), new RavenJObject(), null);
// purpose of doing first backup is to have Database.Document in target directory
// we cancel second backup, when file is already in place
db.Maintenance.StartBackup(BackupDir, true, new DatabaseDocument(), new ResourceBackupState(), CancellationToken.None);
WaitForBackup(db, true);
db.Documents.Put("marcin", null, RavenJObject.Parse("{'email':'*****@*****.**'}"), new RavenJObject(), null);
using (var cts = new CancellationTokenSource())
{
cts.Cancel();
db.Maintenance.StartBackup(BackupDir, true, new DatabaseDocument(), new ResourceBackupState(), cts.Token);
var ex = Assert.Throws <Exception>(() => WaitForBackup(db, true));
Assert.Contains("Backup was canceled", ex.Message);
}
db.Dispose();
IOExtensions.DeleteDirectory(DataDir);
var restoreEx = Assert.Throws <InvalidOperationException>(() =>
{
MaintenanceActions.Restore(new RavenConfiguration(), new DatabaseRestoreRequest
{
BackupLocation = BackupDir,
DatabaseLocation = DataDir,
Defrag = true
}, s => { });
});
Assert.Equal("Backup failure marker was detected. Unable to restore from given directory.", restoreEx.Message);
}
}
public void CanRestoreFullToMultipleLocationsEmbeddedWithIndexing()
{
string storage;
using (var store = NewDocumentStore(runInMemory: false))
{
storage = store.Configuration.DefaultStorageTypeName;
new User_ByName().Execute(store);
using (var sesion = store.OpenSession())
{
sesion.Store(new User {
Name = "Regina"
});
sesion.SaveChanges();
}
WaitForIndexing(store);
store.SystemDatabase.Maintenance.StartBackup(backupDir, false, new DatabaseDocument());
WaitForBackup(store.SystemDatabase, true);
}
MaintenanceActions.Restore(new RavenConfiguration(), new DatabaseRestoreRequest
{
BackupLocation = backupDir,
DatabaseLocation = dataDir,
DatabaseName = "foo",
IndexesLocation = indexesDir,
JournalsLocation = jouranlDir
}, Console.WriteLine);
var ravenConfiguration = new RavenConfiguration
{
DefaultStorageTypeName = storage,
DataDirectory = dataDir,
IndexStoragePath = indexesDir
};
ravenConfiguration.Storage.Esent.JournalsStoragePath = jouranlDir;
ravenConfiguration.Storage.Voron.JournalsStoragePath = jouranlDir;
using (var db = new DocumentDatabase(ravenConfiguration))
{
//db.SpinBackgroundWorkers(); -- indexing disabled here
var q = db.Queries.Query("User/ByName", new IndexQuery(), CancellationToken.None);
Assert.Equal(1, q.TotalResults); // we read the indexed results from the restored data
}
}
public void CanRestoreFullToMultipleLocationsEmbedded()
{
using (var store = NewDocumentStore(runInMemory: false))
{
using (var sesion = store.OpenSession())
{
sesion.Store(new User {
Name = "Regina"
});
sesion.SaveChanges();
}
store.SystemDatabase.Maintenance.StartBackup(backupDir, false, new DatabaseDocument());
WaitForBackup(store.SystemDatabase, true);
}
MaintenanceActions.Restore(new RavenConfiguration(), new DatabaseRestoreRequest
{
BackupLocation = backupDir,
DatabaseLocation = dataDir,
DatabaseName = "foo",
IndexesLocation = indexesDir,
JournalsLocation = jouranlDir
}, Console.WriteLine);
using (var store = NewDocumentStore(runInMemory: false, configureStore: documentStore =>
{
documentStore.Configuration.DataDirectory = dataDir;
documentStore.Configuration.IndexStoragePath = indexesDir;
documentStore.Configuration.Storage.Esent.JournalsStoragePath = jouranlDir;
documentStore.Configuration.Storage.Voron.JournalsStoragePath = jouranlDir;
}))
{
using (var sesion = store.OpenSession())
{
Assert.Equal("Regina", sesion.Load <User>(1).Name);
}
}
}
public void Backup_and_restore_of_system_database_should_work()
{
using (var ravenServer = GetNewServer(runInMemory: false, requestedStorage: "esent"))
using (var _ = NewRemoteDocumentStore(ravenDbServer: ravenServer, databaseName: "fooDB", runInMemory: false))
{
using (var systemDatabaseBackupOperation = new BackupOperation
{
BackupPath = BackupDir,
Database = Constants.SystemDatabase,
ServerUrl = ravenServer.SystemDatabase.Configuration.ServerUrl
})
{
Assert.True(systemDatabaseBackupOperation.InitBackup());
WaitForBackup(ravenServer.SystemDatabase, true);
}
}
Assert.DoesNotThrow(() => MaintenanceActions.Restore(new RavenConfiguration(), new RestoreRequest
{
BackupLocation = BackupDir,
DatabaseLocation = DataDir
}, s => { }));
}
public void AfterRestoreOfIncrementalBackupAllIndexesShouldWork()
{
using (var store = NewDocumentStore(requestedStorage: "esent"))
{
new Users_ByName().Execute(store);
using (var session = store.OpenSession())
{
session.Store(new User {
Name = "Arek", Country = "Poland"
});
session.SaveChanges();
}
WaitForIndexing(store);
store.SystemDatabase.Maintenance.StartBackup(BackupDir, true, new DatabaseDocument());
WaitForBackup(store.SystemDatabase, true);
Thread.Sleep(1000); // incremental tag has seconds precision
using (var session = store.OpenSession())
{
session.Store(new User {
Name = "Ramon", Country = "Spain"
});
session.SaveChanges();
}
WaitForIndexing(store);
store.SystemDatabase.Maintenance.StartBackup(BackupDir, true, new DatabaseDocument());
WaitForBackup(store.SystemDatabase, true);
Thread.Sleep(1000); // incremental tag has seconds precision
new Users_ByNameAndCountry().Execute(store);
WaitForIndexing(store);
store.SystemDatabase.Maintenance.StartBackup(BackupDir, true, new DatabaseDocument());
WaitForBackup(store.SystemDatabase, true);
var output = new StringBuilder();
MaintenanceActions.Restore(new RavenConfiguration
{
Settings =
{
{ Constants.Esent.CircularLog, "false" },
{ Constants.Voron.AllowIncrementalBackups, "true" }
}
}, new DatabaseRestoreRequest
{
BackupLocation = BackupDir,
Defrag = true,
DatabaseLocation = DataDir
}, s => output.Append(s));
Assert.DoesNotContain("error", output.ToString().ToLower());
using (var db = new DocumentDatabase(new RavenConfiguration
{
DataDirectory = DataDir,
Settings =
{
{ Constants.Esent.CircularLog, "false" }
}
}))
{
var indexStats = db.Statistics.Indexes;
Assert.Equal(3, indexStats.Length); // Users/* and Raven/DocumentsByEntityName
QueryResult docs = db.Queries.Query("Users/ByName", new IndexQuery
{
Query = "Name:*",
Start = 0,
PageSize = 10
}, CancellationToken.None);
Assert.Equal(2, docs.Results.Count);
docs = db.Queries.Query("Users/ByNameAndCountry", new IndexQuery
{
Query = "Name:*",
Start = 0,
PageSize = 10
}, CancellationToken.None);
Assert.Equal(2, docs.Results.Count);
}
}
}
public void ShouldNotSetAutoIndexesToAbandonedPriorityAfterDatabaseRecovery()
{
using (var db = new DocumentDatabase(new RavenConfiguration
{
DataDirectory = DataDir,
RunInUnreliableYetFastModeThatIsNotSuitableForProduction = false
}))
{
db.SpinBackgroundWorkers();
db.Indexes.PutIndex(new RavenDocumentsByEntityName().IndexName, new RavenDocumentsByEntityName().CreateIndexDefinition());
db.Documents.Put("users/1", null, RavenJObject.Parse("{'Name':'Arek'}"), RavenJObject.Parse("{'Raven-Entity-Name':'Users'}"), null);
db.Documents.Put("users/2", null, RavenJObject.Parse("{'Name':'David'}"), RavenJObject.Parse("{'Raven-Entity-Name':'Users'}"), null);
var results = db.ExecuteDynamicQuery("Users", new IndexQuery()
{
PageSize = 128,
Start = 0,
Cutoff = SystemTime.UtcNow,
Query = "Name:Arek"
}, CancellationToken.None);
WaitForIndexing(db);
var autoIdexes = db.Statistics.Indexes.Where(x => x.Name.StartsWith("Auto")).ToList();
Assert.True(autoIdexes.Count > 0);
autoIdexes.ForEach(x => db.TransactionalStorage.Batch(accessor => accessor.Indexing.SetIndexPriority(x.Id, IndexingPriority.Idle)));
db.Maintenance.StartBackup(BackupDir, false, new DatabaseDocument());
WaitForBackup(db, true);
}
IOExtensions.DeleteDirectory(DataDir);
MaintenanceActions.Restore(new RavenConfiguration(), new DatabaseRestoreRequest
{
BackupLocation = BackupDir,
DatabaseLocation = DataDir,
Defrag = true
}, s => { });
using (var db = new DocumentDatabase(new RavenConfiguration
{
DataDirectory = DataDir,
RunInUnreliableYetFastModeThatIsNotSuitableForProduction = false,
}))
{
db.SpinBackgroundWorkers();
db.RunIdleOperations();
var autoIndexes = db.Statistics.Indexes.Where(x => x.Name.StartsWith("Auto")).ToList();
Assert.True(autoIndexes.Count > 0);
foreach (var indexStats in autoIndexes)
{
Assert.NotEqual(indexStats.Priority, IndexingPriority.Abandoned);
}
}
}
public DocumentDatabase(InMemoryRavenConfiguration configuration, TransportState transportState = null)
{
DocumentLock = new PutSerialLock();
Name = configuration.DatabaseName;
Configuration = configuration;
this.transportState = transportState ?? new TransportState();
ExtensionsState = new AtomicDictionary <object>();
using (LogManager.OpenMappedContext("database", Name ?? Constants.SystemDatabase))
{
Log.Debug("Start loading the following database: {0}", Name ?? Constants.SystemDatabase);
initializer = new DocumentDatabaseInitializer(this, configuration);
initializer.InitializeEncryption();
initializer.ValidateLicense();
initializer.SubscribeToDomainUnloadOrProcessExit();
initializer.ExecuteAlterConfiguration();
initializer.SatisfyImportsOnce();
backgroundTaskScheduler = configuration.CustomTaskScheduler ?? TaskScheduler.Default;
recentTouches = new SizeLimitedConcurrentDictionary <string, TouchedDocumentInfo>(configuration.MaxRecentTouchesToRemember, StringComparer.OrdinalIgnoreCase);
configuration.Container.SatisfyImportsOnce(this);
workContext = new WorkContext
{
Database = this,
DatabaseName = Name,
IndexUpdateTriggers = IndexUpdateTriggers,
ReadTriggers = ReadTriggers,
TaskScheduler = backgroundTaskScheduler,
Configuration = configuration,
IndexReaderWarmers = IndexReaderWarmers
};
try
{
uuidGenerator = new SequentialUuidGenerator();
initializer.InitializeTransactionalStorage(uuidGenerator);
lastCollectionEtags = new LastCollectionEtags(TransactionalStorage, WorkContext);
}
catch (Exception)
{
if (TransactionalStorage != null)
{
TransactionalStorage.Dispose();
}
throw;
}
try
{
TransactionalStorage.Batch(actions => uuidGenerator.EtagBase = actions.General.GetNextIdentityValue("Raven/Etag"));
// Index codecs must be initialized before we try to read an index
InitializeIndexCodecTriggers();
initializer.InitializeIndexStorage();
Attachments = new AttachmentActions(this, recentTouches, uuidGenerator, Log);
Documents = new DocumentActions(this, recentTouches, uuidGenerator, Log);
Indexes = new IndexActions(this, recentTouches, uuidGenerator, Log);
Maintenance = new MaintenanceActions(this, recentTouches, uuidGenerator, Log);
Notifications = new NotificationActions(this, recentTouches, uuidGenerator, Log);
Patches = new PatchActions(this, recentTouches, uuidGenerator, Log);
Queries = new QueryActions(this, recentTouches, uuidGenerator, Log);
Tasks = new TaskActions(this, recentTouches, uuidGenerator, Log);
Transformers = new TransformerActions(this, recentTouches, uuidGenerator, Log);
inFlightTransactionalState = TransactionalStorage.GetInFlightTransactionalState(Documents.Put, Documents.Delete);
CompleteWorkContextSetup();
prefetcher = new Prefetcher(workContext);
indexingExecuter = new IndexingExecuter(workContext, prefetcher);
RaiseIndexingWiringComplete();
InitializeTriggersExceptIndexCodecs();
SecondStageInitialization();
ExecuteStartupTasks();
lastCollectionEtags.Initialize();
Log.Debug("Finish loading the following database: {0}", configuration.DatabaseName ?? Constants.SystemDatabase);
}
catch (Exception)
{
Dispose();
throw;
}
}
}
public async Task <HttpResponseMessage> Restore()
{
if (EnsureSystemDatabase() == false)
{
return(GetMessageWithString("Restore is only possible from the system database", HttpStatusCode.BadRequest));
}
var restoreStatus = new RestoreStatus {
Messages = new List <string>()
};
var restoreRequest = await ReadJsonObjectAsync <RestoreRequest>();
DatabaseDocument databaseDocument = null;
var databaseDocumentPath = Path.Combine(restoreRequest.BackupLocation, "Database.Document");
if (File.Exists(databaseDocumentPath))
{
var databaseDocumentText = File.ReadAllText(databaseDocumentPath);
databaseDocument = RavenJObject.Parse(databaseDocumentText).JsonDeserialization <DatabaseDocument>();
}
var databaseName = !string.IsNullOrWhiteSpace(restoreRequest.DatabaseName) ? restoreRequest.DatabaseName
: databaseDocument == null ? null : databaseDocument.Id;
if (string.IsNullOrWhiteSpace(databaseName))
{
var errorMessage = (databaseDocument == null || String.IsNullOrWhiteSpace(databaseDocument.Id))
? "Database.Document file is invalid - database name was not found and not supplied in the request (Id property is missing or null). This is probably a bug - should never happen."
: "A database name must be supplied if the restore location does not contain a valid Database.Document file";
restoreStatus.Messages.Add(errorMessage);
DatabasesLandlord.SystemDatabase.Documents.Put(RestoreStatus.RavenRestoreStatusDocumentKey, null, RavenJObject.FromObject(new { restoreStatus }), new RavenJObject(), null);
return(GetMessageWithString(errorMessage, HttpStatusCode.BadRequest));
}
if (databaseName == Constants.SystemDatabase)
{
return(GetMessageWithString("Cannot do an online restore for the <system> database", HttpStatusCode.BadRequest));
}
var ravenConfiguration = new RavenConfiguration
{
DatabaseName = databaseName,
IsTenantDatabase = true
};
if (databaseDocument != null)
{
foreach (var setting in databaseDocument.Settings)
{
ravenConfiguration.Settings[setting.Key] = setting.Value;
}
}
if (File.Exists(Path.Combine(restoreRequest.BackupLocation, BackupMethods.Filename)))
{
ravenConfiguration.DefaultStorageTypeName = typeof(Raven.Storage.Voron.TransactionalStorage).AssemblyQualifiedName;
}
else if (Directory.Exists(Path.Combine(restoreRequest.BackupLocation, "new")))
{
ravenConfiguration.DefaultStorageTypeName = typeof(Raven.Storage.Esent.TransactionalStorage).AssemblyQualifiedName;
}
ravenConfiguration.CustomizeValuesForTenant(databaseName);
ravenConfiguration.Initialize();
string documentDataDir;
ravenConfiguration.DataDirectory = ResolveTenantDataDirectory(restoreRequest.DatabaseLocation, databaseName, out documentDataDir);
restoreRequest.DatabaseLocation = ravenConfiguration.DataDirectory;
DatabasesLandlord.SystemDatabase.Documents.Delete(RestoreStatus.RavenRestoreStatusDocumentKey, null, null);
bool defrag;
if (bool.TryParse(GetQueryStringValue("defrag"), out defrag))
{
restoreRequest.Defrag = defrag;
}
await Task.Factory.StartNew(() =>
{
MaintenanceActions.Restore(ravenConfiguration, restoreRequest,
msg =>
{
restoreStatus.Messages.Add(msg);
DatabasesLandlord.SystemDatabase.Documents.Put(RestoreStatus.RavenRestoreStatusDocumentKey, null,
RavenJObject.FromObject(restoreStatus), new RavenJObject(), null);
});
if (databaseDocument == null)
{
return;
}
databaseDocument.Settings[Constants.RavenDataDir] = documentDataDir;
if (restoreRequest.IndexesLocation != null)
{
databaseDocument.Settings[Constants.RavenIndexPath] = restoreRequest.IndexesLocation;
}
if (restoreRequest.JournalsLocation != null)
{
databaseDocument.Settings[Constants.RavenTxJournalPath] = restoreRequest.JournalsLocation;
}
databaseDocument.Id = databaseName;
DatabasesLandlord.Protect(databaseDocument);
DatabasesLandlord.SystemDatabase.Documents.Put("Raven/Databases/" + databaseName, null, RavenJObject.FromObject(databaseDocument),
new RavenJObject(), null);
restoreStatus.Messages.Add("The new database was created");
DatabasesLandlord.SystemDatabase.Documents.Put(RestoreStatus.RavenRestoreStatusDocumentKey, null,
RavenJObject.FromObject(restoreStatus), new RavenJObject(), null);
}, TaskCreationOptions.LongRunning);
return(GetEmptyMessage());
}
public void AfterFailedRestoreOfIndex_ShouldGenerateWarningAndResetIt(string storageName)
{
using (var db = new DocumentDatabase(new RavenConfiguration
{
DataDirectory = DataDir,
RunInUnreliableYetFastModeThatIsNotSuitableForProduction = false,
DefaultStorageTypeName = storageName,
Settings =
{
{ "Raven/Esent/CircularLog", "false" },
{ "Raven/Voron/AllowIncrementalBackups", "true" }
}
}))
{
db.SpinBackgroundWorkers();
db.Indexes.PutIndex(new RavenDocumentsByEntityName().IndexName, new RavenDocumentsByEntityName().CreateIndexDefinition());
db.Documents.Put("users/1", null, RavenJObject.Parse("{'Name':'Arek'}"), RavenJObject.Parse("{'Raven-Entity-Name':'Users'}"), null);
db.Documents.Put("users/2", null, RavenJObject.Parse("{'Name':'David'}"), RavenJObject.Parse("{'Raven-Entity-Name':'Users'}"), null);
WaitForIndexing(db);
var databaseDocument = new DatabaseDocument();
db.Maintenance.StartBackup(BackupDir, false, databaseDocument);
WaitForBackup(db, true);
db.Documents.Put("users/3", null, RavenJObject.Parse("{'Name':'Daniel'}"), RavenJObject.Parse("{'Raven-Entity-Name':'Users'}"), null);
WaitForIndexing(db);
db.Maintenance.StartBackup(BackupDir, true, databaseDocument);
WaitForBackup(db, true);
}
IOExtensions.DeleteDirectory(DataDir);
var incrementalDirectories = Directory.GetDirectories(BackupDir, "Inc*");
// delete 'index-files.required-for-index-restore' to make backup corrupted according to the reported error
var combine = Directory.GetFiles(incrementalDirectories.First(), "index-files.required-for-index-restore", SearchOption.AllDirectories).First();
File.Delete(combine);
var sb = new StringBuilder();
MaintenanceActions.Restore(new RavenConfiguration(), new RestoreRequest
{
BackupLocation = BackupDir,
DatabaseLocation = DataDir
}, s => sb.Append(s));
Assert.Contains(
"could not be restored. All already copied index files was deleted." +
" Index will be recreated after launching Raven instance",
sb.ToString());
using (var db = new DocumentDatabase(new RavenConfiguration {
DataDirectory = DataDir
}))
{
db.SpinBackgroundWorkers();
QueryResult queryResult;
do
{
queryResult = db.Queries.Query("Raven/DocumentsByEntityName", new IndexQuery
{
Query = "Tag:[[Users]]",
PageSize = 10
}, CancellationToken.None);
} while (queryResult.IsStale);
Assert.Equal(3, queryResult.Results.Count);
}
}
public void AfterBackupRestore_IndexConsistentWithWritesDuringBackup()
{
var count = 1;
var docId = string.Format("ayende{0}", count++.ToString("D4"));
db.Documents.Put(docId, null, RavenJObject.Parse("{'email':'*****@*****.**'}"), RavenJObject.Parse("{'Raven-Entity-Name':'Users'}"), null);
db.SpinBackgroundWorkers();
QueryResult queryResult;
do
{
queryResult = db.Queries.Query("Raven/DocumentsByEntityName", new IndexQuery
{
Query = "Tag:[[Users]]",
PageSize = 10
}, CancellationToken.None);
} while (queryResult.IsStale);
Assert.Equal(1, queryResult.Results.Count);
var runInserts = true;
Task.Run(() =>
{
while (runInserts)
{
docId = string.Format("ayende{0}", count++.ToString("D4"));
db.Documents.Put(docId, null, RavenJObject.Parse("{'email':'*****@*****.**'}"), RavenJObject.Parse("{'Raven-Entity-Name':'Users'}"), null);
db.IndexStorage.FlushMapIndexes();
}
});
db.Maintenance.StartBackup(BackupDir, false, new DatabaseDocument());
WaitForBackup(db, true);
runInserts = false;
db.Dispose();
IOExtensions.DeleteDirectory(DataDir);
MaintenanceActions.Restore(new RavenConfiguration(), new DatabaseRestoreRequest
{
BackupLocation = BackupDir,
DatabaseLocation = DataDir,
Defrag = true
}, s => { });
db = new DocumentDatabase(new RavenConfiguration {
DataDirectory = DataDir
});
docId = string.Format("ayende{0}", count++.ToString("D4"));
db.Documents.Put(docId, null, RavenJObject.Parse("{'email':'*****@*****.**'}"), RavenJObject.Parse("{'Raven-Entity-Name':'Users'}"), null);
db.SpinBackgroundWorkers();
int next = 0;
var storedDocs = new List <string>();
while (true)
{
var batch = db.Documents.GetDocumentsWithIdStartingWith("ayende", null, null, next, 1024, CancellationToken.None, ref next);
storedDocs.AddRange(batch.Select(doc => doc.Value <RavenJObject>("@metadata").Value <string>("@id")));
if (batch.Length < 1024)
{
break;
}
}
List <string> indexedDocs;
bool stale;
do
{
indexedDocs = db.Queries.QueryDocumentIds("Raven/DocumentsByEntityName", new IndexQuery
{
Query = "Tag:[[Users]]",
PageSize = int.MaxValue,
WaitForNonStaleResultsAsOfNow = true
}, new CancellationTokenSource(), out stale).ToList();
} while (stale);
if (storedDocs.Count != indexedDocs.Count)
{
var storedHash = new HashSet <string>(storedDocs);
var indexedHash = new HashSet <string>(indexedDocs);
foreach (var id in storedDocs.Union(indexedDocs).OrderBy(x => x))
{
Debug.WriteLine("{0} Database:{1} Indexed:{2}", id, storedHash.Contains(id), indexedHash.Contains(id));
}
}
Assert.Equal(storedDocs.Count, indexedDocs.Count());
db.Dispose();
}
