public async Task restore_settings_tests()
{
var backupPath = NewDataPath(suffix: "BackupFolder");
using (var store = GetDocumentStore(new Options
{
ModifyDatabaseName = s => $"{s}_2"
}))
{
var restoreConfiguration = new RestoreBackupConfiguration();
var restoreBackupTask = new RestoreBackupOperation(restoreConfiguration);
var e = Assert.Throws <RavenException>(() => store.Maintenance.Server.Send(restoreBackupTask));
Assert.Contains("Database name can't be null or empty", e.InnerException.Message);
restoreConfiguration.DatabaseName = store.Database;
restoreBackupTask = new RestoreBackupOperation(restoreConfiguration);
e = Assert.Throws <RavenException>(() => store.Maintenance.Server.Send(restoreBackupTask));
Assert.Contains("Cannot restore data to an existing database", e.InnerException.Message);
restoreConfiguration.DatabaseName = "test-" + Guid.NewGuid();
restoreBackupTask = new RestoreBackupOperation(restoreConfiguration);
e = Assert.Throws <RavenException>(() => store.Maintenance.Server.Send(restoreBackupTask));
Assert.Contains("Backup location can't be null or empty", e.InnerException.Message);
restoreConfiguration.BackupLocation = "this-path-doesn't-exist\\";
restoreBackupTask = new RestoreBackupOperation(restoreConfiguration);
e = Assert.Throws <RavenException>(() => store.Maintenance.Server.Send(restoreBackupTask));
Assert.Contains("Backup location doesn't exist", e.InnerException.Message);
using (var session = store.OpenAsyncSession())
{
await session.StoreAsync(new User { Name = "oren" }, "users/1");
await session.SaveChangesAsync();
}
var config = new PeriodicBackupConfiguration
{
BackupType = BackupType.Backup,
LocalSettings = new LocalSettings
{
FolderPath = backupPath
},
IncrementalBackupFrequency = "* * * * *" //every minute
};
var backupTaskId = (await store.Maintenance.SendAsync(new UpdatePeriodicBackupOperation(config))).TaskId;
await store.Maintenance.SendAsync(new StartBackupOperation(true, backupTaskId));
var operation = new GetPeriodicBackupStatusOperation(backupTaskId);
SpinWait.SpinUntil(() =>
{
var getPeriodicBackupResult = store.Maintenance.Send(operation);
return(getPeriodicBackupResult.Status?.LastEtag > 0);
}, TimeSpan.FromSeconds(15));
restoreConfiguration.BackupLocation = backupPath;
restoreConfiguration.DataDirectory = backupPath;
restoreBackupTask = new RestoreBackupOperation(restoreConfiguration);
e = Assert.Throws <RavenException>(() => store.Maintenance.Server.Send(restoreBackupTask));
Assert.Contains("New data directory must be empty of any files or folders", e.InnerException.Message);
var emptyFolder = NewDataPath(suffix: "BackupFolderRestore");
restoreConfiguration.BackupLocation = backupPath;
restoreConfiguration.DataDirectory = emptyFolder;
}
}
public async Task periodic_backup_should_work_with_long_intervals()
{
var backupPath = NewDataPath(suffix: "BackupFolder");
using (var store = GetDocumentStore())
{
var periodicBackupRunner = (await GetDocumentDatabaseInstanceFor(store)).PeriodicBackupRunner;
// get by reflection the maxTimerTimeoutInMilliseconds field
// this field is the maximum interval acceptable in .Net's threading timer
// if the requested backup interval is bigger than this maximum interval,
// a timer with maximum interval will be used several times until the interval cumulatively
// will be equal to requested interval
typeof(PeriodicBackupRunner)
.GetField(nameof(PeriodicBackupRunner.MaxTimerTimeout), BindingFlags.Instance | BindingFlags.Public)
.SetValue(periodicBackupRunner, TimeSpan.FromMilliseconds(100));
using (var session = store.OpenAsyncSession())
{
await session.StoreAsync(new User { Name = "oren 1" }, "users/1");
await session.SaveChangesAsync();
}
var config = new PeriodicBackupConfiguration
{
BackupType = BackupType.Backup,
LocalSettings = new LocalSettings
{
FolderPath = backupPath
},
IncrementalBackupFrequency = "* * * * *" //every minute
};
var operation = new UpdatePeriodicBackupOperation(config, store.Database);
var result = await store.Admin.Server.SendAsync(operation);
var periodicBackupTaskId = result.TaskId;
SpinWait.SpinUntil(() =>
{
var getPeriodicBackupStatus = new GetPeriodicBackupStatusOperation(store.Database, periodicBackupTaskId);
var status = store.Admin.Server.Send(getPeriodicBackupStatus).Status;
return(status?.LastFullBackup != null);
}, 2000);
using (var session = store.OpenAsyncSession())
{
await session.StoreAsync(new User { Name = "oren 2" }, "users/2");
await session.SaveChangesAsync();
}
SpinWait.SpinUntil(() =>
{
var getPeriodicBackupStatus =
new GetPeriodicBackupStatusOperation(store.Database, periodicBackupTaskId);
var status = store.Admin.Server.Send(getPeriodicBackupStatus).Status;
return(status?.LastFullBackup != null && status.LastIncrementalBackup != null);
}, TimeSpan.FromMinutes(2));
}
using (var store = GetDocumentStore(new Options
{
ModifyDatabaseName = s => $"{s}_2"
}))
{
await store.Smuggler.ImportIncrementalAsync(new DatabaseSmugglerImportOptions(),
Directory.GetDirectories(backupPath).First());
using (var session = store.OpenAsyncSession())
{
var user = await session.LoadAsync <User>("users/1");
Assert.Equal("oren 1", user.Name);
user = await session.LoadAsync <User>("users/2");
Assert.Equal("oren 2", user.Name);
}
}
}
public async Task can_restore_smuggler_correctly()
{
var backupPath = NewDataPath(suffix: "BackupFolder");
using (var store = GetDocumentStore())
{
using (var session = store.OpenAsyncSession())
{
await session.StoreAsync(new User { Name = "oren" }, "users/1");
await session.SaveChangesAsync();
}
var config = new PeriodicBackupConfiguration
{
LocalSettings = new LocalSettings
{
FolderPath = backupPath
},
IncrementalBackupFrequency = "* * * * *" //every minute
};
var result = await store.Admin.Server.SendAsync(new UpdatePeriodicBackupOperation(config, store.Database));
var periodicBackupTaskId = result.TaskId;
var operation = new GetPeriodicBackupStatusOperation(store.Database, periodicBackupTaskId);
SpinWait.SpinUntil(() =>
{
var getPeriodicBackupResult = store.Admin.Server.Send(operation);
return(getPeriodicBackupResult.Status?.LastEtag > 0);
}, TimeSpan.FromMinutes(2));
var etagForBackups = store.Admin.Server.Send(operation).Status.LastEtag;
using (var session = store.OpenAsyncSession())
{
await session.StoreAsync(new User { Name = "ayende" }, "users/2");
await session.SaveChangesAsync();
}
SpinWait.SpinUntil(() =>
{
var newLastEtag = store.Admin.Server.Send(operation).Status.LastEtag;
return(newLastEtag != etagForBackups);
}, TimeSpan.FromMinutes(2));
}
using (var store1 = GetDocumentStore(new Options
{
ModifyDatabaseName = s => $"{s}_2"
}))
using (var store2 = GetDocumentStore(new Options
{
ModifyDatabaseName = s => $"{s}_2"
}))
{
var backupDirectory = Directory.GetDirectories(backupPath).First();
var backupToMovePath = $"{backupPath}\\IncrementalBackupTemp";
Directory.CreateDirectory(backupToMovePath);
var incrementalBackupFile = Directory.GetFiles(backupDirectory).Last();
var fileName = Path.GetFileName(incrementalBackupFile);
File.Move(incrementalBackupFile, $"{backupToMovePath}\\{fileName}");
await store1.Smuggler.ImportIncrementalAsync(new DatabaseSmugglerImportOptions(), backupDirectory);
using (var session = store1.OpenAsyncSession())
{
var users = await session.LoadAsync <User>(new[] { "users/1", "users/2" });
var keyValuePair = users.First();
Assert.NotNull(keyValuePair.Value);
Assert.True(keyValuePair.Value.Name == "oren" && keyValuePair.Key == "users/1");
Assert.Null(users.Last().Value);
}
await store2.Smuggler.ImportIncrementalAsync(new DatabaseSmugglerImportOptions(), backupToMovePath);
using (var session = store2.OpenAsyncSession())
{
var users = await session.LoadAsync <User>(new[] { "users/1", "users/2" });
Assert.Null(users.First().Value);
var keyValuePair = users.Last();
Assert.NotNull(keyValuePair.Value);
Assert.True(keyValuePair.Value.Name == "ayende" && keyValuePair.Key == "users/2");
}
}
}
public async Task CreateFullAndIncrementalBackupWithIndexInTheMiddle()
{
var backupPath = NewDataPath(suffix: "BackupFolder");
using (var store = GetDocumentStore())
{
using (var session = store.OpenSession())
{
session.Store(new User
{
Name = "Toli"
}, "users/1");
session.SaveChanges();
}
var config = new PeriodicBackupConfiguration
{
LocalSettings = new LocalSettings
{
FolderPath = backupPath
},
Name = "full",
FullBackupFrequency = "* */6 * * *",
BackupType = BackupType.Backup
};
var result = await store.Maintenance.SendAsync(new UpdatePeriodicBackupOperation(config));
var documentDatabase = (await GetDocumentDatabaseInstanceFor(store));
RunBackup(result.TaskId, documentDatabase, true, store);
var input = new IndexDefinition
{
Name = "Users_ByName",
Maps =
{
"from user in docs.Users select new { user.Name }"
},
Type = IndexType.Map
};
await store
.Maintenance
.SendAsync(new PutIndexesOperation(new[] { input }));
config.IncrementalBackupFrequency = "* */300 * * *";
config.TaskId = result.TaskId;
result = await store.Maintenance.SendAsync(new UpdatePeriodicBackupOperation(config));
RunBackup(result.TaskId, documentDatabase, false, store);
var backupDirectory = Directory.GetDirectories(backupPath).First();
var databaseName = GetDatabaseName() + "restore";
var files = Directory.GetFiles(backupDirectory)
.Where(BackupUtils.IsBackupFile)
.OrderBackups()
.ToArray();
RestoreBackupConfiguration config2 = new RestoreBackupConfiguration()
{
BackupLocation = backupDirectory,
DatabaseName = databaseName,
LastFileNameToRestore = files.Last()
};
RestoreBackupOperation restoreOperation = new RestoreBackupOperation(config2);
store.Maintenance.Server.Send(restoreOperation)
.WaitForCompletion(TimeSpan.FromSeconds(30));
using (var store2 = GetDocumentStore(new Options()
{
CreateDatabase = false,
ModifyDatabaseName = s => databaseName
}))
{
var stats = store2.Maintenance.ForDatabase(databaseName).Send(new GetStatisticsOperation());
Assert.Equal(1, stats.CountOfIndexes);
Assert.Equal(1, stats.CountOfDocuments);
};
}
}
public async Task SkipExportingTheServerWideBackup2()
{
var backupPath = NewDataPath(suffix: "BackupFolder");
using (var store = GetDocumentStore())
{
var serverWideBackupConfiguration = new ServerWideBackupConfiguration
{
Disabled = false,
FullBackupFrequency = "0 2 * * 0",
IncrementalBackupFrequency = "0 2 * * 1",
LocalSettings = new LocalSettings
{
FolderPath = backupPath
}
};
await store.Maintenance.Server.SendAsync(new PutServerWideBackupConfigurationOperation(serverWideBackupConfiguration));
var record = await store.Maintenance.Server.SendAsync(new GetDatabaseRecordOperation(store.Database));
var backup = record.PeriodicBackups.First();
var backupTaskId = backup.TaskId;
// save another backup task in the database record
var backupConfiguration = new PeriodicBackupConfiguration
{
Disabled = true,
FullBackupFrequency = "0 2 * * 0",
IncrementalBackupFrequency = "0 2 * * 1"
};
await store.Maintenance.SendAsync(new UpdatePeriodicBackupOperation(backupConfiguration));
await store.Maintenance.SendAsync(new StartBackupOperation(true, backupTaskId));
string backupDirectory = null;
var value = WaitForValue(() =>
{
var status = store.Maintenance.Send(new GetPeriodicBackupStatusOperation(backupTaskId)).Status;
backupDirectory = status?.LocalBackup.BackupDirectory;
return(status?.LastEtag);
}, 0);
Assert.Equal(0, value);
var files = Directory.GetFiles(backupDirectory)
.Where(BackupUtils.IsBackupFile)
.OrderBackups()
.ToArray();
var databaseName = GetDatabaseName() + "restore";
var restoreConfig = new RestoreBackupConfiguration
{
BackupLocation = backupDirectory,
DatabaseName = databaseName,
LastFileNameToRestore = files.OrderBackups().Last()
};
var restoreOperation = new RestoreBackupOperation(restoreConfig);
store.Maintenance.Server.Send(restoreOperation)
.WaitForCompletion(TimeSpan.FromSeconds(30));
// old server should have 2: 1 server wide and 1 regular backup
using (var store2 = GetDocumentStore(new Options
{
CreateDatabase = false,
ModifyDatabaseName = s => databaseName,
}))
{
var record2 = await store.Maintenance.Server.SendAsync(new GetDatabaseRecordOperation(store2.Database));
Assert.Equal(2, record2.PeriodicBackups.Count);
}
// new server should have only one backup
var server = GetNewServer();
using (var store3 = GetDocumentStore(new Options
{
CreateDatabase = false,
ModifyDatabaseName = s => databaseName,
Server = server
}))
{
store3.Maintenance.Server.Send(restoreOperation)
.WaitForCompletion(TimeSpan.FromSeconds(30));
var record3 = await store3.Maintenance.Server.SendAsync(new GetDatabaseRecordOperation(databaseName));
Assert.Equal(1, record3.PeriodicBackups.Count);
}
}
}
private async Task CanDeleteBackupsByDate(
int backupAgeInSeconds,
int numberOfBackupsToCreate,
Action <PeriodicBackupConfiguration, string> modifyConfiguration,
Func <string, Task <int> > getDirectoriesCount,
int timeout, bool checkIncremental = false)
{
var minimumBackupAgeToKeep = TimeSpan.FromSeconds(backupAgeInSeconds);
using (var store = GetDocumentStore())
{
var config = new PeriodicBackupConfiguration
{
IncrementalBackupFrequency = "30 3 L * ?",
RetentionPolicy = new RetentionPolicy
{
MinimumBackupAgeToKeep = minimumBackupAgeToKeep
}
};
modifyConfiguration(config, store.Database);
var backupTaskId = (await store.Maintenance.SendAsync(new UpdatePeriodicBackupOperation(config))).TaskId;
var lastEtag = 0L;
var userId = "";
for (var i = 0; i < numberOfBackupsToCreate; i++)
{
using (var session = store.OpenAsyncSession())
{
var user = new User {
Name = "Grisha"
};
await session.StoreAsync(user);
userId = user.Id;
await session.SaveChangesAsync();
}
// create full backup
lastEtag = await CreateBackup(store, true, backupTaskId, lastEtag, timeout);
using (var session = store.OpenAsyncSession())
{
var user = await session.LoadAsync <User>(userId);
user.Age = 33;
await session.SaveChangesAsync();
}
// create incremental backup
lastEtag = await CreateBackup(store, false, backupTaskId, lastEtag, timeout);
}
await Task.Delay(minimumBackupAgeToKeep + TimeSpan.FromSeconds(5));
if (checkIncremental)
{
using (var session = store.OpenAsyncSession())
{
var user = await session.LoadAsync <User>(userId);
user.Name = "Egor";
user.Age = 322;
await session.SaveChangesAsync();
}
// create incremental backup with retention policy
lastEtag = await CreateBackup(store, false, backupTaskId, lastEtag, timeout);
}
using (var session = store.OpenAsyncSession())
{
await session.StoreAsync(new User { Name = "Grisha" });
await session.SaveChangesAsync();
}
await CreateBackup(store, true, backupTaskId, lastEtag, timeout);
var directoriesCount = await getDirectoriesCount(store.Database);
var expectedNumberOfDirectories = checkIncremental ? 2 : 1;
Assert.Equal(expectedNumberOfDirectories, directoriesCount);
}
}
public async Task CreateFullAndIncrementalBackupWithCompareExchangeInTheMiddle()
{
var backupPath = NewDataPath(suffix: "BackupFolder");
using (var store = GetDocumentStore())
{
using (var session = store.OpenSession())
{
session.Store(new User
{
Name = "Toli"
}, "users/1");
session.SaveChanges();
}
var config = new PeriodicBackupConfiguration
{
LocalSettings = new LocalSettings
{
FolderPath = backupPath
},
Name = "full",
FullBackupFrequency = "* */6 * * *",
BackupType = BackupType.Backup
};
var result = await store.Maintenance.SendAsync(new UpdatePeriodicBackupOperation(config));
var documentDatabase = (await GetDocumentDatabaseInstanceFor(store));
RunBackup(result.TaskId, documentDatabase, true, store);
CompareExchangeResult <string> compareExchangeResult
= store.Operations.Send(
new PutCompareExchangeValueOperation <string>("users/1", "Mitzi", 0));
WaitForValue(() => compareExchangeResult.Successful, true);
config.IncrementalBackupFrequency = "* */2 * * *";
config.TaskId = result.TaskId;
result = await store.Maintenance.SendAsync(new UpdatePeriodicBackupOperation(config));
RunBackup(result.TaskId, documentDatabase, false, store);
compareExchangeResult
= store.Operations.Send(
new PutCompareExchangeValueOperation <string>("users/1", "Mitzi2", compareExchangeResult.Index));
WaitForValue(() => compareExchangeResult.Successful, true);
config.IncrementalBackupFrequency = "* */2 * * *";
config.TaskId = result.TaskId;
result = await store.Maintenance.SendAsync(new UpdatePeriodicBackupOperation(config));
RunBackup(result.TaskId, documentDatabase, false, store);
var backupDirectory = Directory.GetDirectories(backupPath).First();
var databaseName = GetDatabaseName() + "restore";
var files = Directory.GetFiles(backupDirectory)
.Where(BackupUtils.IsBackupFile)
.OrderBackups()
.ToArray();
RestoreBackupConfiguration config2 = new RestoreBackupConfiguration()
{
BackupLocation = backupDirectory,
DatabaseName = databaseName,
LastFileNameToRestore = files.Last()
};
RestoreBackupOperation restoreOperation = new RestoreBackupOperation(config2);
store.Maintenance.Server.Send(restoreOperation)
.WaitForCompletion();
using (var store2 = GetDocumentStore(new Options()
{
CreateDatabase = false,
ModifyDatabaseName = s => databaseName
}))
{
using (var session = store2.OpenSession())
{
var doc = session.Load <User>("users/1");
Assert.NotNull(doc);
};
CompareExchangeValue <string> readResult =
store2.Operations.Send(new GetCompareExchangeValueOperation <string>("users/1"));
Assert.Equal("Mitzi2", readResult.Value);
};
}
}
public async Task CanStoreAndEditServerWideBackupForIdleDatabase()
{
using var server = GetNewServer(new ServerCreationOptions
{
CustomSettings = new Dictionary <string, string>
{
[RavenConfiguration.GetKey(x => x.Databases.MaxIdleTime)] = "10",
[RavenConfiguration.GetKey(x => x.Databases.FrequencyToCheckForIdle)] = "3",
[RavenConfiguration.GetKey(x => x.Core.RunInMemory)] = "false"
}
});
using (var store = GetDocumentStore(new Options {
Server = server, RunInMemory = false
}))
{
var fullFreq = "0 2 1 1 *";
var incFreq = "0 2 * * 0";
var putConfiguration = new ServerWideBackupConfiguration
{
FullBackupFrequency = fullFreq,
IncrementalBackupFrequency = incFreq,
LocalSettings = new LocalSettings
{
FolderPath = "test/folder"
}
};
Assert.Equal(1, WaitForValue(() => server.ServerStore.IdleDatabases.Count, 1, timeout: 60000, interval: 1000));
var result = await store.Maintenance.Server.SendAsync(new PutServerWideBackupConfigurationOperation(putConfiguration));
var serverWideConfiguration = await store.Maintenance.Server.SendAsync(new GetServerWideBackupConfigurationOperation(result.Name));
Assert.NotNull(serverWideConfiguration);
Assert.Equal(fullFreq, serverWideConfiguration.FullBackupFrequency);
Assert.Equal(incFreq, serverWideConfiguration.IncrementalBackupFrequency);
Assert.Equal(1, server.ServerStore.IdleDatabases.Count);
// update the backup configuration
putConfiguration.Name = serverWideConfiguration.Name;
putConfiguration.TaskId = serverWideConfiguration.TaskId;
putConfiguration.FullBackupFrequency = "0 2 * * 0";
var oldName = result.Name;
result = await store.Maintenance.Server.SendAsync(new PutServerWideBackupConfigurationOperation(putConfiguration));
Exception ex = null;
try
{
await server.ServerStore.Cluster.WaitForIndexNotification(result.RaftCommandIndex, TimeSpan.FromMinutes(1));
}
catch (Exception e)
{
ex = e;
}
finally
{
Assert.Null(ex);
}
var record = await store.Maintenance.Server.SendAsync(new GetDatabaseRecordOperation(store.Database));
Assert.Equal(1, record.PeriodicBackups.Count);
PeriodicBackupConfiguration periodicBackupConfiguration = record.PeriodicBackups.First();
var newServerWideConfiguration = await store.Maintenance.Server.SendAsync(new GetServerWideBackupConfigurationOperation(result.Name));
// compare with periodic backup task
Assert.NotEqual(newServerWideConfiguration.TaskId, periodicBackupConfiguration.TaskId); // backup task id in db record doesn't change
Assert.Equal(PutServerWideBackupConfigurationCommand.GetTaskNameForDatabase(oldName), periodicBackupConfiguration.Name);
Assert.Equal(incFreq, periodicBackupConfiguration.FullBackupFrequency);
Assert.Equal(incFreq, periodicBackupConfiguration.IncrementalBackupFrequency);
Assert.NotEqual(serverWideConfiguration.FullBackupFrequency, periodicBackupConfiguration.FullBackupFrequency);
// compare with previous server wide backup
Assert.NotEqual(serverWideConfiguration.TaskId, newServerWideConfiguration.TaskId); // task id in server storage get increased with each change
Assert.Equal(oldName, result.Name);
Assert.Equal(incFreq, newServerWideConfiguration.FullBackupFrequency);
Assert.Equal(incFreq, newServerWideConfiguration.IncrementalBackupFrequency);
Assert.NotEqual(serverWideConfiguration.FullBackupFrequency, newServerWideConfiguration.FullBackupFrequency);
Assert.Equal(1, server.ServerStore.IdleDatabases.Count);
}
}
public async Task CanPassNodeTagToRestoreBackupOperation()
{
var myBackupsList = new List <MyBackup>();
var backupPath = NewDataPath(suffix: "BackupFolder");
var clusterSize = 3;
var databaseName = GetDatabaseName();
(List <RavenServer> Nodes, RavenServer Leader)cluster = await CreateRaftCluster(clusterSize, false, useSsl : false);
using (var store = new DocumentStore
{
Urls = new[] { cluster.Leader.WebUrl },
Database = databaseName,
}.Initialize())
{
var doc = new DatabaseRecord(databaseName);
var databaseResult = await store.Maintenance.Server.SendAsync(new CreateDatabaseOperation(doc, clusterSize));
await store.Maintenance.SendAsync(new CreateSampleDataOperation());
var myNodesList = databaseResult.Topology.AllNodes.ToList();
Assert.True(clusterSize == myNodesList.Count, $"clusterSize({clusterSize}) == myNodesList.Count({myNodesList.Count})");
foreach (var node in myNodesList)
{
var myGuid = Guid.NewGuid();
var backupConfig = new PeriodicBackupConfiguration
{
LocalSettings = new LocalSettings
{
FolderPath = Path.Combine(backupPath, myGuid.ToString())
},
FullBackupFrequency = "0 0 1 1 *", // once a year on 1st january at 00:00
BackupType = BackupType.Backup,
Name = $"Task_{node}_{myGuid}",
MentorNode = node
};
var result = await store.Maintenance.SendAsync(new UpdatePeriodicBackupOperation(backupConfig));
await WaitForRaftIndexToBeAppliedOnClusterNodes(result.RaftCommandIndex, cluster.Nodes);
OngoingTask res = null;
await WaitForValueAsync(async() =>
{
res = await store.Maintenance.SendAsync(new GetOngoingTaskInfoOperation(result.TaskId, OngoingTaskType.Backup));
return(res.ResponsibleNode.NodeTag);
}, node, interval : 333);
Assert.NotNull(res);
Assert.True(node == res.MentorNode, $"node({node}) == res.MentorNode({res.MentorNode})");
Assert.True(node == res.ResponsibleNode.NodeTag, $"node({node}) == res.ResponsibleNode.NodeTag({res.ResponsibleNode.NodeTag})");
myBackupsList.Add(new MyBackup
{
BackupTaskId = result.TaskId,
Guid = myGuid,
NodeTag = res.ResponsibleNode.NodeTag
});
}
foreach (var myBackup in myBackupsList)
{
var res = await store.Maintenance.SendAsync(new StartBackupOperation(isFullBackup : true, myBackup.BackupTaskId));
Assert.True(myBackup.NodeTag == res.Result.ResponsibleNode, $"myBackup.NodeTag({myBackup.NodeTag}) == res.ResponsibleNode({res.Result.ResponsibleNode})");
var operation = new GetPeriodicBackupStatusOperation(myBackup.BackupTaskId);
PeriodicBackupStatus status = null;
var value = WaitForValue(() =>
{
status = store.Maintenance.Send(operation).Status;
if (status?.LastEtag == null)
{
return(false);
}
return(true);
}, true);
Assert.True(value, $"Got status: {status != null}, exception: {status?.Error?.Exception}, LocalBackup.Exception: {status?.LocalBackup?.Exception}");
Assert.True(status != null, $"status != null, exception: {status?.Error?.Exception}, LocalBackup.Exception: {status?.LocalBackup?.Exception}");
Assert.True(myBackup.NodeTag == status.NodeTag, $"myBackup.NodeTag({myBackup.NodeTag}) == status.NodeTag({status.NodeTag})");
var prePath = Path.Combine(backupPath, myBackup.Guid.ToString());
myBackup.BackupPath = Path.Combine(prePath, status.FolderName);
}
var dbs = new List <string>();
foreach (var myBackup in myBackupsList)
{
var name = $"restored_DB1_{myBackup.NodeTag}";
var restoreConfig = new RestoreBackupConfiguration
{
DatabaseName = name,
BackupLocation = myBackup.BackupPath
};
dbs.Add(name);
var restoreBackupTask = store.Maintenance.Server.Send(new RestoreBackupOperation(restoreConfig, myBackup.NodeTag));
restoreBackupTask.WaitForCompletion(TimeSpan.FromSeconds(30));
}
var dbNames = await store.Maintenance.Server.SendAsync(new GetDatabaseNamesOperation(0, int.MaxValue));
Assert.Equal(clusterSize + 1, dbNames.Length);
dbs.ForEach(db => Assert.True(dbNames.Contains(db), $"numOfDbs.Contains(db), db = {db}"));
}
}
public async Task CanToggleTaskState()
{
var clusterSize = 3;
var databaseName = "TestDB";
var leader = await CreateRaftClusterAndGetLeader(clusterSize);
ModifyOngoingTaskResult addWatcherRes;
UpdatePeriodicBackupOperationResult updateBackupResult;
using (var store = new DocumentStore
{
Urls = new[] { leader.WebUrl },
Database = databaseName
}.Initialize())
{
var doc = new DatabaseRecord(databaseName);
var databaseResult = await store.Maintenance.Server.SendAsync(new CreateDatabaseOperation(doc, clusterSize));
Assert.Equal(clusterSize, databaseResult.Topology.AllNodes.Count());
foreach (var server in Servers)
{
await server.ServerStore.Cluster.WaitForIndexNotification(databaseResult.RaftCommandIndex);
}
foreach (var server in Servers)
{
await server.ServerStore.DatabasesLandlord.TryGetOrCreateResourceStore(databaseName);
}
var watcher = new ExternalReplication("Watcher1", "Connection");
addWatcherRes = await AddWatcherToReplicationTopology((DocumentStore)store, watcher, new[] { "http://127.0.0.1:9090" });
var backupConfig = new PeriodicBackupConfiguration
{
LocalSettings = new LocalSettings
{
FolderPath = NewDataPath(suffix: "BackupFolder")
},
FullBackupFrequency = "* */1 * * *",
IncrementalBackupFrequency = "* */2 * * *",
Disabled = true
};
updateBackupResult = await store.Maintenance.SendAsync(new UpdatePeriodicBackupOperation(backupConfig));
}
using (var store = new DocumentStore
{
Urls = new[] { leader.WebUrl },
Database = databaseName,
Conventions =
{
DisableTopologyUpdates = true
}
}.Initialize())
{
var taskId = addWatcherRes.TaskId;
var op = new ToggleOngoingTaskStateOperation(taskId, OngoingTaskType.Replication, true);
await store.Maintenance.SendAsync(op);
var result = await GetTaskInfo((DocumentStore)store, taskId, OngoingTaskType.Replication);
Assert.Equal(OngoingTaskState.Disabled, result.TaskState);
taskId = updateBackupResult.TaskId;
op = new ToggleOngoingTaskStateOperation(taskId, OngoingTaskType.Backup, false);
await store.Maintenance.SendAsync(op);
result = await GetTaskInfo((DocumentStore)store, taskId, OngoingTaskType.Backup);
Assert.Equal(OngoingTaskState.Enabled, result.TaskState);
}
}
public async Task CanGetTaskInfo()
{
var clusterSize = 3;
var databaseName = "TestDB";
var leader = await CreateRaftClusterAndGetLeader(clusterSize);
ModifyOngoingTaskResult addWatcherRes;
UpdatePeriodicBackupOperationResult updateBackupResult;
AddEtlOperationResult addRavenEtlResult;
AddEtlOperationResult addSqlEtlResult;
RavenEtlConfiguration etlConfiguration;
SqlEtlConfiguration sqlConfiguration;
ExternalReplication watcher;
SqlConnectionString sqlConnectionString;
var sqlScript = @"
var orderData = {
Id: __document_id,
OrderLinesCount: this.OrderLines.length,
TotalCost: 0
};
loadToOrders(orderData);
";
using (var store = new DocumentStore
{
Urls = new[] { leader.WebUrl },
Database = databaseName
}.Initialize())
{
var doc = new DatabaseRecord(databaseName);
var databaseResult = await store.Maintenance.Server.SendAsync(new CreateDatabaseOperation(doc, clusterSize));
Assert.Equal(clusterSize, databaseResult.Topology.AllNodes.Count());
foreach (var server in Servers)
{
await server.ServerStore.Cluster.WaitForIndexNotification(databaseResult.RaftCommandIndex);
}
foreach (var server in Servers)
{
await server.ServerStore.DatabasesLandlord.TryGetOrCreateResourceStore(databaseName);
}
watcher = new ExternalReplication("Watcher1", "Connection")
{
Name = "MyExternalReplication"
};
addWatcherRes = await AddWatcherToReplicationTopology((DocumentStore)store, watcher, new[] { "http://127.0.0.1:9090" });
var backupConfig = new PeriodicBackupConfiguration
{
Name = "backup1",
LocalSettings = new LocalSettings
{
FolderPath = NewDataPath(suffix: "BackupFolder")
},
AzureSettings = new AzureSettings
{
StorageContainer = "abc"
},
FullBackupFrequency = "* */1 * * *",
IncrementalBackupFrequency = "* */2 * * *",
Disabled = true
};
updateBackupResult = await store.Maintenance.SendAsync(new UpdatePeriodicBackupOperation(backupConfig));
store.Maintenance.Send(new PutConnectionStringOperation <RavenConnectionString>(new RavenConnectionString
{
Name = "cs",
TopologyDiscoveryUrls = new [] { "http://127.0.0.1:8080" },
Database = "Northwind",
}));
etlConfiguration = new RavenEtlConfiguration()
{
Name = "tesst",
ConnectionStringName = "cs",
Transforms =
{
new Transformation()
{
Name = "loadAll",
Collections ={ "Users" },
Script = "loadToUsers(this)"
}
}
};
addRavenEtlResult = store.Maintenance.Send(new AddEtlOperation <RavenConnectionString>(etlConfiguration));
sqlConnectionString = new SqlConnectionString
{
Name = "abc",
ConnectionString = @"Data Source=localhost\sqlexpress;Integrated Security=SSPI;Connection Timeout=3" + $";Initial Catalog=SqlReplication-{store.Database};",
FactoryName = "System.Data.SqlClient"
};
store.Maintenance.Send(new PutConnectionStringOperation <SqlConnectionString>(sqlConnectionString));
sqlConfiguration = new SqlEtlConfiguration()
{
Name = "abc",
ConnectionStringName = "abc",
SqlTables =
{
new SqlEtlTable {
TableName = "Orders", DocumentIdColumn = "Id", InsertOnlyMode = false
},
new SqlEtlTable {
TableName = "OrderLines", DocumentIdColumn = "OrderId", InsertOnlyMode = false
},
},
Transforms =
{
new Transformation()
{
Name = "OrdersAndLines",
Collections = new List <string>{
"Orders"
},
Script = sqlScript
}
}
};
addSqlEtlResult = store.Maintenance.Send(new AddEtlOperation <SqlConnectionString>(sqlConfiguration));
}
using (var store = new DocumentStore
{
Urls = new[] { leader.WebUrl },
Database = databaseName,
Conventions =
{
DisableTopologyUpdates = true
}
}.Initialize())
{
var taskId = addWatcherRes.TaskId;
var replicationResult = (OngoingTaskReplication) await GetTaskInfo((DocumentStore)store, taskId, OngoingTaskType.Replication);
Assert.Equal(watcher.Database, replicationResult.DestinationDatabase);
Assert.Equal(watcher.Url, replicationResult.DestinationUrl);
Assert.Equal(watcher.Name, replicationResult.TaskName);
taskId = updateBackupResult.TaskId;
var backupResult = (OngoingTaskBackup) await GetTaskInfo((DocumentStore)store, taskId, OngoingTaskType.Backup);
Assert.Equal("Local", backupResult.BackupDestinations[0]);
Assert.Equal("Azure", backupResult.BackupDestinations[1]);
Assert.Equal("backup1", backupResult.TaskName);
Assert.Equal(OngoingTaskState.Disabled, backupResult.TaskState);
taskId = addRavenEtlResult.TaskId;
var etlResult = (OngoingTaskRavenEtlDetails) await GetTaskInfo((DocumentStore)store, taskId, OngoingTaskType.RavenEtl);
Assert.Equal("cs", etlResult.Configuration.ConnectionStringName);
Assert.Equal("tesst", etlResult.Configuration.Name);
Assert.Equal("loadAll", etlResult.Configuration.Transforms[0].Name);
Assert.Equal("loadToUsers(this)", etlResult.Configuration.Transforms[0].Script);
Assert.Equal("Users", etlResult.Configuration.Transforms[0].Collections[0]);
Assert.Equal(etlConfiguration.Name, etlResult?.TaskName);
taskId = addSqlEtlResult.TaskId;
var sqlResult = (OngoingTaskSqlEtlDetails) await GetTaskInfo((DocumentStore)store, taskId, OngoingTaskType.SqlEtl);
Assert.Equal("abc", sqlResult.Configuration.ConnectionStringName);
Assert.Equal("abc", sqlResult.Configuration.Name);
Assert.Equal("OrdersAndLines", sqlResult.Configuration.Transforms[0].Name);
Assert.Equal(sqlScript, sqlResult.Configuration.Transforms[0].Script);
Assert.Equal("Orders", sqlResult.Configuration.Transforms[0].Collections[0]);
Assert.NotNull(sqlResult.Configuration.SqlTables);
Assert.Equal(sqlConfiguration.Name, sqlResult?.TaskName);
}
}
public async Task can_restore_legacy_backup()
{
var backupPath = NewDataPath(suffix: "BackupFolder");
using (var store = GetDocumentStore())
{
using (var session = store.OpenAsyncSession())
{
await session.StoreAsync(new User { Name = "oren" }, "users/1");
await session.SaveChangesAsync();
}
var config = new PeriodicBackupConfiguration
{
LocalSettings = new LocalSettings
{
FolderPath = backupPath
},
IncrementalBackupFrequency = "* * * * *" //every minute
};
var backupTaskId = (await store.Maintenance.SendAsync(new UpdatePeriodicBackupOperation(config))).TaskId;
await store.Maintenance.SendAsync(new StartBackupOperation(true, backupTaskId));
var operation = new GetPeriodicBackupStatusOperation(backupTaskId);
var value = WaitForValue(() =>
{
var status = store.Maintenance.Send(operation).Status;
return(status?.LastEtag);
}, 1);
Assert.Equal(1, value);
using (var session = store.OpenAsyncSession())
{
await session.StoreAsync(new User { Name = "ayende" }, "users/2");
await session.SaveChangesAsync();
}
var lastEtag = store.Maintenance.Send(new GetStatisticsOperation()).LastDocEtag;
await store.Maintenance.SendAsync(new StartBackupOperation(false, backupTaskId));
value = WaitForValue(() => store.Maintenance.Send(operation).Status.LastEtag, lastEtag);
Assert.Equal(lastEtag, value);
}
using (var store = GetDocumentStore(new Options
{
ModifyDatabaseName = s => $"{s}_2"
}))
{
var directory = Directory.GetDirectories(backupPath).First();
var files = Directory.GetFiles(directory);
Assert.Equal(2, files.Length);
File.Move(files[0], files[0].Replace(Constants.Documents.PeriodicBackup.FullBackupExtension, ".ravendb-full-dump"));
File.Move(files[1], files[1].Replace(Constants.Documents.PeriodicBackup.IncrementalBackupExtension, ".ravendb-incremental-dump"));
await store.Smuggler.ImportIncrementalAsync(new DatabaseSmugglerImportOptions(),
directory);
using (var session = store.OpenAsyncSession())
{
var users = await session.LoadAsync <User>(new[] { "users/1", "users/2" });
Assert.True(users.Any(x => x.Value.Name == "oren"));
Assert.True(users.Any(x => x.Value.Name == "ayende"));
}
}
}
public async Task can_backup_to_directory_multiple_backups_with_long_interval()
{
var backupPath = NewDataPath(suffix: "BackupFolder");
using (var store = GetDocumentStore())
{
var periodicBackupRunner = (await GetDocumentDatabaseInstanceFor(store)).PeriodicBackupRunner;
// get by reflection the maxTimerTimeoutInMilliseconds field
// this field is the maximum interval acceptable in .Net's threading timer
// if the requested backup interval is bigger than this maximum interval,
// a timer with maximum interval will be used several times until the interval cumulatively
// will be equal to requested interval
typeof(PeriodicBackupRunner)
.GetField(nameof(PeriodicBackupRunner.MaxTimerTimeout), BindingFlags.Instance | BindingFlags.Public)
.SetValue(periodicBackupRunner, TimeSpan.FromMilliseconds(100));
using (var session = store.OpenAsyncSession())
{
await session.StoreAsync(new User { Name = "oren" }, "users/1");
await session.SaveChangesAsync();
}
var config = new PeriodicBackupConfiguration
{
LocalSettings = new LocalSettings
{
FolderPath = backupPath
},
IncrementalBackupFrequency = "* * * * *" //every minute
};
var backupTaskId = (await store.Maintenance.SendAsync(new UpdatePeriodicBackupOperation(config))).TaskId;
await store.Maintenance.SendAsync(new StartBackupOperation(true, backupTaskId));
var operation = new GetPeriodicBackupStatusOperation(backupTaskId);
SpinWait.SpinUntil(() =>
{
var getPeriodicBackupResult = store.Maintenance.Send(operation);
return(getPeriodicBackupResult.Status?.LastEtag > 0);
}, TimeSpan.FromSeconds(15));
var etagForBackups = store.Maintenance.Send(operation).Status.LastEtag;
using (var session = store.OpenAsyncSession())
{
await session.StoreAsync(new User { Name = "ayende" }, "users/2");
await session.SaveChangesAsync();
}
await store.Maintenance.SendAsync(new StartBackupOperation(false, backupTaskId));
SpinWait.SpinUntil(() =>
{
var newLastEtag = store.Maintenance.Send(operation).Status.LastEtag;
return(newLastEtag != etagForBackups);
}, TimeSpan.FromSeconds(15));
}
using (var store = GetDocumentStore(new Options
{
ModifyDatabaseName = s => $"{s}_2"
}))
{
await store.Smuggler.ImportIncrementalAsync(new DatabaseSmugglerImportOptions(),
Directory.GetDirectories(backupPath).First());
using (var session = store.OpenAsyncSession())
{
var users = await session.LoadAsync <User>(new[] { "users/1", "users/2" });
Assert.True(users.Any(x => x.Value.Name == "oren"));
Assert.True(users.Any(x => x.Value.Name == "ayende"));
}
}
}
public async Task preorid_backup_should_export_starting_from_last_etag()
{
//http://issues.hibernatingrhinos.com/issue/RavenDB-11395
var backupPath = NewDataPath(suffix: "BackupFolder");
using (var store = GetDocumentStore())
{
using (var session = store.OpenAsyncSession())
{
await session.StoreAsync(new User { Name = "oren" }, "users/1");
await session.StoreAsync(new User { Name = "aviv" }, "users/2");
session.CountersFor("users/1").Increment("likes", 100);
session.CountersFor("users/2").Increment("downloads", 200);
await session.SaveChangesAsync();
}
var config = new PeriodicBackupConfiguration
{
LocalSettings = new LocalSettings
{
FolderPath = backupPath
},
IncrementalBackupFrequency = "* * * * *" //every minute
};
var backupTaskId = (await store.Maintenance.SendAsync(new UpdatePeriodicBackupOperation(config))).TaskId;
await store.Maintenance.SendAsync(new StartBackupOperation(true, backupTaskId));
var operation = new GetPeriodicBackupStatusOperation(backupTaskId);
SpinWait.SpinUntil(() =>
{
var getPeriodicBackupResult = store.Maintenance.Send(operation);
return(getPeriodicBackupResult.Status?.LastEtag > 0);
}, TimeSpan.FromSeconds(15));
var exportPath = GetBackupPath(store, backupTaskId, incremental: false);
using (var store2 = GetDocumentStore())
{
await store2.Smuggler.ImportAsync(new DatabaseSmugglerImportOptions(), exportPath);
var stats = await store2.Maintenance.SendAsync(new GetStatisticsOperation());
Assert.Equal(2, stats.CountOfDocuments);
Assert.Equal(2, stats.CountOfCounters);
using (var session = store2.OpenAsyncSession())
{
var user1 = await session.LoadAsync <User>("users/1");
var user2 = await session.LoadAsync <User>("users/2");
Assert.Equal("oren", user1.Name);
Assert.Equal("aviv", user2.Name);
var dic = await session.CountersFor(user1).GetAllAsync();
Assert.Equal(1, dic.Count);
Assert.Equal(100, dic["likes"]);
dic = await session.CountersFor(user2).GetAllAsync();
Assert.Equal(1, dic.Count);
Assert.Equal(200, dic["downloads"]);
}
}
var etagForBackups = store.Maintenance.Send(operation).Status.LastEtag;
using (var session = store.OpenAsyncSession())
{
await session.StoreAsync(new User { Name = "ayende" }, "users/3");
session.CountersFor("users/3").Increment("votes", 300);
await session.SaveChangesAsync();
}
await store.Maintenance.SendAsync(new StartBackupOperation(false, backupTaskId));
SpinWait.SpinUntil(() =>
{
var newLastEtag = store.Maintenance.Send(operation).Status.LastEtag;
return(newLastEtag != etagForBackups);
}, TimeSpan.FromSeconds(15));
exportPath = GetBackupPath(store, backupTaskId);
using (var store3 = GetDocumentStore())
{
// importing to a new database, in order to verify that
// periodic backup imports only the changed documents (and counters)
await store3.Smuggler.ImportAsync(new DatabaseSmugglerImportOptions(), exportPath);
var stats = await store3.Maintenance.SendAsync(new GetStatisticsOperation());
Assert.Equal(1, stats.CountOfDocuments);
Assert.Equal(1, stats.CountOfCounters);
using (var session = store3.OpenAsyncSession())
{
var user3 = await session.LoadAsync <User>("users/3");
Assert.Equal("ayende", user3.Name);
var dic = await session.CountersFor(user3).GetAllAsync();
Assert.Equal(1, dic.Count);
Assert.Equal(300, dic["votes"]);
}
}
}
}
private long CreateBackupTask(PeriodicBackup periodicBackup, bool isFullBackup)
{
using (periodicBackup.UpdateBackupTask())
{
if (periodicBackup.Disposed)
{
throw new InvalidOperationException("Backup task was already disposed");
}
if (periodicBackup.RunningTask != null)
{
return(periodicBackup.RunningBackupTaskId ?? -1);
}
if (_serverStore.Server.CpuCreditsBalance.BackgroundTasksAlertRaised.IsRaised())
{
throw new BackupDelayException(
$"Failed to start Backup Task: '{periodicBackup.Configuration.Name}'. " +
$"The task cannot run because the CPU credits allocated to this machine are nearing exhaustion.")
{
DelayPeriod = _serverStore.Configuration.Server.CpuCreditsExhaustionBackupDelay.AsTimeSpan
};
}
if (LowMemoryNotification.Instance.LowMemoryState)
{
throw new BackupDelayException(
$"Failed to start Backup Task: '{periodicBackup.Configuration.Name}'. " +
$"The task cannot run because the server is in low memory state.")
{
DelayPeriod = _serverStore.Configuration.Backup.LowMemoryBackupDelay.AsTimeSpan
};
}
if (_serverStore.ConcurrentBackupsSemaphore.Wait(0) == false)
{
throw new BackupDelayException(
$"Failed to start Backup Task: '{periodicBackup.Configuration.Name}'. " +
$"The task exceeds the maximum number of concurrent backup tasks configured. " +
$"Current value of Backup.MaxNumberOfConcurrentBackups is: {_serverStore.Configuration.Backup.MaxNumberOfConcurrentBackups:#,#;;0}")
{
DelayPeriod = TimeSpan.FromMinutes(1)
};
}
try
{
var backupStatus = periodicBackup.BackupStatus = GetBackupStatus(periodicBackup.Configuration.TaskId, periodicBackup.BackupStatus);
var backupToLocalFolder = PeriodicBackupConfiguration.CanBackupUsing(periodicBackup.Configuration.LocalSettings);
// check if we need to do a new full backup
if (backupStatus.LastFullBackup == null || // no full backup was previously performed
backupStatus.NodeTag != _serverStore.NodeTag || // last backup was performed by a different node
backupStatus.BackupType != periodicBackup.Configuration.BackupType || // backup type has changed
backupStatus.LastEtag == null || // last document etag wasn't updated
backupToLocalFolder && BackupTask.DirectoryContainsBackupFiles(backupStatus.LocalBackup.BackupDirectory, IsFullBackupOrSnapshot) == false)
// the local folder already includes a full backup or snapshot
{
isFullBackup = true;
}
var operationId = _database.Operations.GetNextOperationId();
var backupTypeText = GetBackupTypeText(isFullBackup, periodicBackup.Configuration.BackupType);
periodicBackup.StartTime = SystemTime.UtcNow;
var backupTask = new BackupTask(
_serverStore,
_database,
periodicBackup,
isFullBackup,
backupToLocalFolder,
operationId,
_tempBackupPath,
_logger,
_cancellationToken.Token);
periodicBackup.RunningBackupTaskId = operationId;
periodicBackup.CancelToken = backupTask.TaskCancelToken;
var backupTaskName = $"{backupTypeText} backup task: '{periodicBackup.Configuration.Name}'";
var task = _database.Operations.AddOperation(
null,
backupTaskName,
Operations.Operations.OperationType.DatabaseBackup,
taskFactory: onProgress => Task.Run(async() =>
{
try
{
using (_database.PreventFromUnloading())
{
return(await backupTask.RunPeriodicBackup(onProgress));
}
}
finally
{
_serverStore.ConcurrentBackupsSemaphore.Release();
periodicBackup.RunningTask = null;
periodicBackup.RunningBackupTaskId = null;
periodicBackup.CancelToken = null;
periodicBackup.RunningBackupStatus = null;
if (periodicBackup.HasScheduledBackup() &&
_cancellationToken.IsCancellationRequested == false)
{
var newBackupTimer = GetTimer(periodicBackup.Configuration, periodicBackup.BackupStatus);
periodicBackup.UpdateTimer(newBackupTimer, discardIfDisabled: true);
}
}
}, backupTask.TaskCancelToken.Token),
id: operationId,
token: backupTask.TaskCancelToken);
periodicBackup.RunningTask = task;
task.ContinueWith(_ => backupTask.TaskCancelToken.Dispose());
return(operationId);
}
catch (Exception e)
{
// releasing the semaphore because we failed to start the backup task
_serverStore.ConcurrentBackupsSemaphore.Release();
var message = $"Failed to start the backup task: '{periodicBackup.Configuration.Name}'";
if (_logger.IsOperationsEnabled)
{
_logger.Operations(message, e);
}
_database.NotificationCenter.Add(AlertRaised.Create(
_database.Name,
$"Periodic Backup task: '{periodicBackup.Configuration.Name}'",
message,
AlertType.PeriodicBackup,
NotificationSeverity.Error,
details: new ExceptionDetails(e)));
throw;
}
}
}
public async Task can_backup_and_restore_snapshot()
{
var defaultS3Settings = GetS3Settings();
using (var store = GetDocumentStore())
{
using (var session = store.OpenAsyncSession())
{
await session.StoreAsync(new User { Name = "oren" }, "users/1");
await session.SaveChangesAsync();
}
using (var session = store.OpenAsyncSession())
{
await session
.Query<User>()
.Where(x => x.Name == "oren")
.ToListAsync(); // create an index to backup
await session
.Query<Order>()
.Where(x => x.Freight > 20)
.ToListAsync(); // create an index to backup
session.CountersFor("users/1").Increment("likes", 100); //create a counter to backup
await session.SaveChangesAsync();
}
var config = new PeriodicBackupConfiguration
{
BackupType = BackupType.Snapshot,
S3Settings = defaultS3Settings,
IncrementalBackupFrequency = "* * * * *" //every minute
};
var backupTaskId = (await store.Maintenance.SendAsync(new UpdatePeriodicBackupOperation(config))).TaskId;
await store.Maintenance.SendAsync(new StartBackupOperation(true, backupTaskId));
var operation = new GetPeriodicBackupStatusOperation(backupTaskId);
var value = WaitForValue(() =>
{
var status = store.Maintenance.Send(operation).Status;
return status?.LastEtag;
}, 4);
Assert.Equal(4, value);
var backupStatus = store.Maintenance.Send(operation);
using (var session = store.OpenAsyncSession())
{
await session.StoreAsync(new User { Name = "ayende" }, "users/2");
await session.StoreAsync(new User { Name = "ayende" }, "users/3");
session.CountersFor("users/2").Increment("downloads", 200);
await session.SaveChangesAsync();
}
var lastEtag = store.Maintenance.Send(new GetStatisticsOperation()).LastDocEtag;
await store.Maintenance.SendAsync(new StartBackupOperation(false, backupTaskId));
value = WaitForValue(() => store.Maintenance.Send(operation).Status.LastEtag, lastEtag);
Assert.Equal(lastEtag, value);
// restore the database with a different name
string restoredDatabaseName = $"restored_database_snapshot-{Guid.NewGuid()}";
var subfolderS3Settings = GetS3Settings(backupStatus.Status.FolderName);
using (RestoreDatabaseFromCloud(store,
new RestoreFromS3Configuration
{
DatabaseName = restoredDatabaseName,
Settings = subfolderS3Settings
},
TimeSpan.FromSeconds(60)))
{
using (var session = store.OpenAsyncSession(restoredDatabaseName))
{
var users = await session.LoadAsync<User>(new[] { "users/1", "users/2" });
Assert.NotNull(users["users/1"]);
Assert.NotNull(users["users/2"]);
Assert.True(users.Any(x => x.Value.Name == "oren"));
Assert.True(users.Any(x => x.Value.Name == "ayende"));
var val = await session.CountersFor("users/1").GetAsync("likes");
Assert.Equal(100, val);
val = await session.CountersFor("users/2").GetAsync("downloads");
Assert.Equal(200, val);
}
var stats = await store.Maintenance.SendAsync(new GetStatisticsOperation());
Assert.Equal(2, stats.CountOfIndexes);
var originalDatabase = await Server.ServerStore.DatabasesLandlord.TryGetOrCreateResourceStore(store.Database);
var restoredDatabase = await Server.ServerStore.DatabasesLandlord.TryGetOrCreateResourceStore(restoredDatabaseName);
using (restoredDatabase.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext ctx))
using (ctx.OpenReadTransaction())
{
var databaseChangeVector = DocumentsStorage.GetDatabaseChangeVector(ctx);
Assert.Equal($"A:8-{originalDatabase.DbBase64Id}, A:10-{restoredDatabase.DbBase64Id}", databaseChangeVector);
}
}
}
}
/// <summary>
/// Check whether RavenDB is running
/// Check whether or not a database already exists with the DBName
/// Set up auto-backup of the database
/// Ensure that all guilds shared with the bot have been added to the database
/// </summary>
/// <param name="client">The discord client</param>
public static async void DatabaseInitialise(DiscordSocketClient client)
{
/*
* if (Process.GetProcesses().FirstOrDefault(x => x.ProcessName == "Raven.Server") == null)
* {
* Logger.LogMessage("RavenDB: Server isn't running. Please make sure RavenDB is running.\nExiting ...", LogSeverity.Critical);
* await Task.Delay(5000);
* Environment.Exit(Environment.ExitCode);
* }
*/
if (!Ping(Config.Load().ServerURL))
{
Logger.LogMessage("RavenDB: Server isn't running. Please make sure RavenDB is running.\nExiting ...", LogSeverity.Critical);
await Task.Delay(5000);
Environment.Exit(Environment.ExitCode);
}
var dbinitialised = false;
if (Store.Maintenance.Server.Send(new GetDatabaseNamesOperation(0, 5)).All(x => x != DBName))
{
await Store.Maintenance.Server.SendAsync(new CreateDatabaseOperation(new DatabaseRecord(DBName)));
Logger.LogMessage($"Created Database {DBName}.");
dbinitialised = true;
}
Logger.LogMessage("RavenDB: Setting up backup operation...");
var newbackup = new PeriodicBackupConfiguration
{
Name = "Backup",
BackupType = BackupType.Backup,
FullBackupFrequency = "0 */6 * * *",
IncrementalBackupFrequency = "0 2 * * *",
LocalSettings = new LocalSettings {
FolderPath = Path.Combine(AppContext.BaseDirectory, "setup/backups/")
}
};
var Record = Store.Maintenance.ForDatabase(DBName).Server.Send(new GetDatabaseRecordOperation(DBName));
var backupop = Record.PeriodicBackups.FirstOrDefault(x => x.Name == "Backup");
try
{
if (backupop == null)
{
await Store.Maintenance.ForDatabase(DBName).SendAsync(new UpdatePeriodicBackupOperation(newbackup)).ConfigureAwait(false);
}
else
{
//In the case that we already have a backup operation setup, ensure that we update the backup location accordingly
backupop.LocalSettings = new LocalSettings {
FolderPath = Path.Combine(AppContext.BaseDirectory, "setup/backups/")
};
await Store.Maintenance.ForDatabase(DBName).SendAsync(new UpdatePeriodicBackupOperation(backupop));
}
}
catch
{
Logger.LogMessage("RavenDB: Error setting up backup operation, they may not be saved\n", LogSeverity.Warning);
}
if (dbinitialised)
{
return;
}
using (var session = Store.OpenSession(DBName))
{
try
{
//Check to see wether or not we can actually load the Guilds List saved in our RavenDB
var _ = session.Query <GuildModel.Guild>().ToList();
}
catch
{
//In the case that the check fails, ensure we initalise all servers that contain the bot.
var glist = client.Guilds.Select(x => new GuildModel.Guild
{
GuildID = x.Id
}).ToList();
foreach (var gobj in glist)
{
session.Store(gobj, gobj.GuildID.ToString());
}
session.SaveChanges();
}
}
}
public async Task incremental_and_full_check_last_file_for_backup()
{
var defaultS3Settings = GetS3Settings();
using (var store = GetDocumentStore())
{
using (var session = store.OpenAsyncSession())
{
await session.StoreAsync(new User { Name = "user-1" }, "users/1");
await session.SaveChangesAsync();
}
var config = new PeriodicBackupConfiguration
{
BackupType = BackupType.Backup,
S3Settings = defaultS3Settings,
IncrementalBackupFrequency = "0 */6 * * *",
};
var backupTaskId = (await store.Maintenance.SendAsync(new UpdatePeriodicBackupOperation(config))).TaskId;
await store.Maintenance.SendAsync(new StartBackupOperation(true, backupTaskId));
var operation = new GetPeriodicBackupStatusOperation(backupTaskId);
var value = WaitForValue(() =>
{
var getPeriodicBackupResult = store.Maintenance.Send(operation);
return getPeriodicBackupResult.Status?.LastEtag;
}, 1);
Assert.Equal(1, value);
using (var session = store.OpenAsyncSession())
{
await session.StoreAsync(new User { Name = "user-2" }, "users/2");
await session.SaveChangesAsync();
}
var lastEtag = store.Maintenance.Send(new GetStatisticsOperation()).LastDocEtag;
await store.Maintenance.SendAsync(new StartBackupOperation(false, backupTaskId));
value = WaitForValue(() => store.Maintenance.Send(operation).Status.LastEtag, lastEtag);
Assert.Equal(lastEtag, value);
string lastFileToRestore;
var backupStatus = store.Maintenance.Send(operation);
using (var client = new RavenAwsS3Client(AmazonS3FactAttribute.S3Settings))
{
var fullBackupPath = $"{defaultS3Settings.RemoteFolderName}/{backupStatus.Status.FolderName}";
lastFileToRestore = (await client.ListObjectsAsync(fullBackupPath, string.Empty, false)).FileInfoDetails.Last().FullPath;
}
using (var session = store.OpenAsyncSession())
{
await session.StoreAsync(new User { Name = "user-3" }, "users/3");
await session.SaveChangesAsync();
}
lastEtag = store.Maintenance.Send(new GetStatisticsOperation()).LastDocEtag;
await store.Maintenance.SendAsync(new StartBackupOperation(false, backupTaskId));
value = WaitForValue(() => store.Maintenance.Send(operation).Status.LastEtag, lastEtag);
Assert.Equal(lastEtag, value);
backupStatus = store.Maintenance.Send(operation);
var databaseName = $"restored_database-{Guid.NewGuid()}";
var subfolderS3Settings = GetS3Settings(backupStatus.Status.FolderName);
using (RestoreDatabaseFromCloud(store,
new RestoreFromS3Configuration
{
Settings = subfolderS3Settings,
DatabaseName = databaseName,
LastFileNameToRestore = lastFileToRestore
}))
{
using (var session = store.OpenSession(databaseName))
{
var users = session.Load<User>("users/1");
Assert.NotNull(users);
users = session.Load<User>("users/2");
Assert.NotNull(users);
users = session.Load<User>("users/3");
Assert.Null(users);
}
}
}
}
/// <summary>
/// Initializes the database for use
/// </summary>
/// <returns>
/// The Document Store
/// </returns>
public Task <IDocumentStore> InitializeAsync()
{
// Ensure that the bots database settings are setup, if not prompt to enter details
if (!File.Exists("setup/DBConfig.json"))
{
LogHandler.LogMessage("Please enter details about your bot and database configuration. NOTE: You can hit enter for a default value. ");
LogHandler.LogMessage("Enter the database Name: (ie. MyRavenDatabase) DEFAULT: RavenBOT");
var databaseName = Console.ReadLine();
if (string.IsNullOrEmpty(databaseName))
{
databaseName = "RavenBOT";
}
LogHandler.LogMessage("Enter the database URL: (typically http://127.0.0.1:8080 if hosting locally) DEFAULT: http://127.0.0.1:8080");
var databaseUrl = Console.ReadLine();
if (string.IsNullOrEmpty(databaseUrl))
{
databaseUrl = "http://127.0.0.1:8080";
}
File.WriteAllText("setup/DBConfig.json", JsonConvert.SerializeObject(
new DatabaseObject
{
Name = databaseName,
Urls = new List <string> {
databaseUrl
}
},
Formatting.Indented),
Encoding.UTF8);
Settings = JsonConvert.DeserializeObject <DatabaseObject>(File.ReadAllText("setup/DBConfig.json"));
}
else
{
Settings = JsonConvert.DeserializeObject <DatabaseObject>(File.ReadAllText("setup/DBConfig.json"));
LogHandler.LogMessage("Connecting to Server\n" + $"=> URL: \n{string.Join("\n", Settings.Urls)}\n" + $"=> Name: {Settings.Name}");
}
if (Settings.Urls.Count == 0)
{
throw new Exception("No database urls have been detected in config, please add one");
}
// This initializes the document store, and ensures that RavenDB is working properly
Store = new Lazy <IDocumentStore>(() => new DocumentStore {
Database = Settings.Name, Urls = Settings.Urls.ToArray()
}.Initialize(), true).Value;
if (Store == null)
{
LogHandler.LogMessage("Failed to build document store.", LogSeverity.Critical);
}
// This creates the database
if (Store.Maintenance.Server.Send(new GetDatabaseNamesOperation(0, 20)).All(x => x != Settings.Name))
{
Store.Maintenance.Server.Send(new CreateDatabaseOperation(new DatabaseRecord(Settings.Name)));
LogHandler.LogMessage($"Created Database => {Settings.Name}");
}
// To ensure the backup operation is functioning and backing up to our bots directory we update the backup operation on each boot of the bot
try
{
var record = Store.Maintenance.ForDatabase(Settings.Name).Server.Send(new GetDatabaseRecordOperation(Settings.Name));
var backup = record.PeriodicBackups.FirstOrDefault(x => x.Name == "Backup");
if (backup == null)
{
var newbackup = new PeriodicBackupConfiguration {
Name = "Backup", BackupType = BackupType.Backup, FullBackupFrequency = Settings.FullBackup, IncrementalBackupFrequency = Settings.IncrementalBackup, LocalSettings = new LocalSettings {
FolderPath = Settings.BackupFolder
}
};
Store.Maintenance.ForDatabase(Settings.Name).Send(new UpdatePeriodicBackupOperation(newbackup));
}
else
{
// In the case that we already have a backup operation setup, ensure that we update the backup location accordingly
backup.LocalSettings = new LocalSettings {
FolderPath = Settings.BackupFolder
};
Store.Maintenance.ForDatabase(Settings.Name).Send(new UpdatePeriodicBackupOperation(backup));
}
}
catch
{
LogHandler.LogMessage("RavenDB: Failed to set Backup operation. Backups may not be saved", LogSeverity.Warning);
}
// Prompt the user to set up the bots configuration.
if (Settings.IsConfigCreated == false)
{
var configModel = new ConfigModel();
LogHandler.LogMessage("Enter bots token: (You can get this from https://discordapp.com/developers/applications/me)");
var token = Console.ReadLine();
if (string.IsNullOrWhiteSpace(token))
{
throw new Exception("You must supply a token for this bot to operate.");
}
LogHandler.LogMessage("Enter bots prefix: (This will be used to initiate a command, ie. +say or +help) DEFAULT: +");
var prefix = Console.ReadLine();
if (string.IsNullOrEmpty(prefix))
{
prefix = "+";
}
configModel.Token = token;
configModel.Prefix = prefix;
// This inserts the config object into the database and writes the DatabaseConfig to file.
Execute <ConfigModel>(Operation.CREATE, configModel, "Config");
Settings.IsConfigCreated = true;
File.WriteAllText("setup/DBConfig.json", JsonConvert.SerializeObject(Settings, Formatting.Indented));
}
LogHandler.PrintApplicationInformation(Settings, Execute <ConfigModel>(Operation.LOAD, null, "Config"));
// Note the logger has to be updated/re-set after we set the database up otherwise there will be a null reference when trying to log initially
if (Settings.LogToDatabase)
{
LogHandler.Log = new LoggerConfiguration().MinimumLevel.Is(LogHandler.DiscordLogToEventLevel(Settings.LogSeverity)).WriteTo.Console().WriteTo.RavenDB(
Store,
defaultDatabase: Settings.Name,
expiration: TimeSpan.FromDays(2)).CreateLogger();
}
else
{
LogHandler.Log = new LoggerConfiguration().MinimumLevel.Is(LogHandler.DiscordLogToEventLevel(Settings.LogSeverity)).CreateLogger();
}
return(Task.FromResult(Store));
}
public async Task incremental_and_full_backup_encrypted_db_and_restore_to_encrypted_DB_with_provided_key()
{
var defaultS3Settings = GetS3Settings();
var key = EncryptedServer(out var certificates, out string dbName);
using (var store = GetDocumentStore(new Options
{
AdminCertificate = certificates.ServerCertificate.Value,
ClientCertificate = certificates.ServerCertificate.Value,
ModifyDatabaseName = s => dbName,
ModifyDatabaseRecord = record => record.Encrypted = true,
Path = NewDataPath()
}))
{
using (var session = store.OpenAsyncSession())
{
await session.StoreAsync(new User
{
Name = "oren"
}, "users/1");
await session.SaveChangesAsync();
}
var config = new PeriodicBackupConfiguration
{
BackupType = BackupType.Backup,
S3Settings = defaultS3Settings,
IncrementalBackupFrequency = "0 */6 * * *",
BackupEncryptionSettings = new BackupEncryptionSettings
{
Key = "OI7Vll7DroXdUORtc6Uo64wdAk1W0Db9ExXXgcg5IUs=",
EncryptionMode = EncryptionMode.UseProvidedKey
}
};
var backupTaskId = (await store.Maintenance.SendAsync(new UpdatePeriodicBackupOperation(config))).TaskId;
await store.Maintenance.SendAsync(new StartBackupOperation(true, backupTaskId));
var operation = new GetPeriodicBackupStatusOperation(backupTaskId);
var value = WaitForValue(() =>
{
var getPeriodicBackupResult = store.Maintenance.Send(operation);
return getPeriodicBackupResult.Status?.LastEtag;
}, 1);
Assert.Equal(1, value);
using (var session = store.OpenAsyncSession())
{
await session.StoreAsync(new User { Name = "ayende" }, "users/2");
await session.StoreAsync(new User { Name = "ayende" }, "users/3");
session.CountersFor("users/2").Increment("downloads", 200);
await session.SaveChangesAsync();
}
var lastEtag = store.Maintenance.Send(new GetStatisticsOperation()).LastDocEtag;
await store.Maintenance.SendAsync(new StartBackupOperation(false, backupTaskId));
value = WaitForValue(() => store.Maintenance.Send(operation).Status.LastEtag, lastEtag);
Assert.Equal(lastEtag, value);
var backupStatus = store.Maintenance.Send(operation);
var databaseName = $"restored_database-{Guid.NewGuid()}";
var subfolderS3Settings = GetS3Settings(backupStatus.Status.FolderName);
using (RestoreDatabaseFromCloud(store, new RestoreFromS3Configuration
{
Settings = subfolderS3Settings,
DatabaseName = databaseName,
EncryptionKey = "OI7Vll7DroXdUORtc6Uo64wdAk1W0Db9ExXXgcg5IUs=",
BackupEncryptionSettings = new BackupEncryptionSettings
{
Key = "OI7Vll7DroXdUORtc6Uo64wdAk1W0Db9ExXXgcg5IUs=",
EncryptionMode = EncryptionMode.UseProvidedKey
}
}))
{
using (var session = store.OpenSession(databaseName))
{
var users = session.Load<User>("users/1");
Assert.NotNull(users);
users = session.Load<User>("users/2");
Assert.NotNull(users);
}
}
}
}
public async Task CreateFullAndIncrementalBackupWithIdentitiesInTheMiddle()
{
var backupPath = NewDataPath(suffix: "BackupFolder");
using (var store = GetDocumentStore())
{
using (var session = store.OpenSession())
{
session.Store(new User
{
Id = "users|",
Name = "Toli"
});
session.Store(new User
{
Id = "users|",
Name = "Mitzi"
});
session.SaveChanges();
}
var config = new PeriodicBackupConfiguration
{
LocalSettings = new LocalSettings
{
FolderPath = backupPath
},
Name = "full",
FullBackupFrequency = "* */6 * * *",
BackupType = BackupType.Backup
};
var result = await store.Maintenance.SendAsync(new UpdatePeriodicBackupOperation(config));
var documentDatabase = (await GetDocumentDatabaseInstanceFor(store));
RunBackup(result.TaskId, documentDatabase, true, store);
Dictionary <string, long> identities = store.Maintenance.Send(new GetIdentitiesOperation());
config.IncrementalBackupFrequency = "* */2 * * *";
config.TaskId = result.TaskId;
result = await store.Maintenance.SendAsync(new UpdatePeriodicBackupOperation(config));
RunBackup(result.TaskId, documentDatabase, false, store);
var backupDirectory = Directory.GetDirectories(backupPath).First();
var databaseName = GetDatabaseName() + "restore";
var files = Directory.GetFiles(backupDirectory)
.Where(BackupUtils.IsBackupFile)
.OrderBackups()
.ToArray();
RestoreBackupConfiguration config2 = new RestoreBackupConfiguration()
{
BackupLocation = backupDirectory,
DatabaseName = databaseName,
LastFileNameToRestore = files.Last()
};
RestoreBackupOperation restoreOperation = new RestoreBackupOperation(config2);
store.Maintenance.Server.Send(restoreOperation)
.WaitForCompletion();
using (var store2 = GetDocumentStore(new Options()
{
CreateDatabase = false,
ModifyDatabaseName = s => databaseName
}))
{
Dictionary <string, long> identities2 = store2.Maintenance.Send(new GetIdentitiesOperation());
Assert.Equal(identities.First().Key, identities2.First().Key);
Assert.Equal(identities.First().Value, identities2.First().Value);
};
}
}
public async Task snapshot_encrypted_db_and_restore_to_encrypted_DB()
{
var backupPath = NewDataPath(suffix: "BackupFolder");
var key = EncryptedServer(out var certificates, out string dbName);
var defaultS3Settings = GetS3Settings();
using (var store = GetDocumentStore(new Options
{
AdminCertificate = certificates.ServerCertificate.Value,
ClientCertificate = certificates.ServerCertificate.Value,
ModifyDatabaseName = s => dbName,
ModifyDatabaseRecord = record => record.Encrypted = true,
Path = NewDataPath()
}))
{
using (var session = store.OpenAsyncSession())
{
await session.StoreAsync(new User
{
Name = "oren"
}, "users/1");
await session.SaveChangesAsync();
}
var config = new PeriodicBackupConfiguration
{
BackupType = BackupType.Snapshot,
S3Settings = defaultS3Settings,
IncrementalBackupFrequency = "0 */6 * * *"
};
var backupTaskId = (await store.Maintenance.SendAsync(new UpdatePeriodicBackupOperation(config))).TaskId;
await store.Maintenance.SendAsync(new StartBackupOperation(true, backupTaskId));
var operation = new GetPeriodicBackupStatusOperation(backupTaskId);
var value = WaitForValue(() =>
{
var getPeriodicBackupResult = store.Maintenance.Send(operation);
return getPeriodicBackupResult.Status?.LastEtag;
}, 1);
Assert.Equal(1, value);
var backupStatus = store.Maintenance.Send(operation);
var databaseName = $"restored_database-{Guid.NewGuid()}";
var subfolderS3Settings = GetS3Settings(backupStatus.Status.FolderName);
using (RestoreDatabaseFromCloud(store, new RestoreFromS3Configuration
{
Settings = subfolderS3Settings,
DatabaseName = databaseName,
EncryptionKey = key,
BackupEncryptionSettings = new BackupEncryptionSettings
{
EncryptionMode = EncryptionMode.UseDatabaseKey
}
}))
{
using (var session = store.OpenSession(databaseName))
{
var users = session.Load<User>("users/1");
Assert.NotNull(users);
}
}
}
}
private long CreateBackupTask(PeriodicBackup periodicBackup, bool isFullBackup)
{
if (periodicBackup.UpdateBackupTaskSemaphore.Wait(0) == false)
{
return(periodicBackup.RunningBackupTaskId ?? -1);
}
try
{
if (periodicBackup.RunningTask != null)
{
return(periodicBackup.RunningBackupTaskId ?? -1);
}
var backupStatus = periodicBackup.BackupStatus = GetBackupStatus(periodicBackup.Configuration.TaskId, periodicBackup.BackupStatus);
var backupToLocalFolder = PeriodicBackupConfiguration.CanBackupUsing(periodicBackup.Configuration.LocalSettings);
// check if we need to do a new full backup
if (backupStatus.LastFullBackup == null || // no full backup was previously performed
backupStatus.NodeTag != _serverStore.NodeTag || // last backup was performed by a different node
backupStatus.BackupType != periodicBackup.Configuration.BackupType || // backup type has changed
backupStatus.LastEtag == null || // last document etag wasn't updated
backupToLocalFolder && BackupTask.DirectoryContainsBackupFiles(backupStatus.LocalBackup.BackupDirectory, IsFullBackupOrSnapshot) == false)
// the local folder already includes a full backup or snapshot
{
isFullBackup = true;
}
var operationId = _database.Operations.GetNextOperationId();
var backupTypeText = GetBackupTypeText(isFullBackup, periodicBackup.Configuration.BackupType);
periodicBackup.StartTime = SystemTime.UtcNow;
var backupTask = new BackupTask(
_serverStore,
_database,
periodicBackup,
isFullBackup,
backupToLocalFolder,
operationId,
_tempBackupPath,
_logger,
_cancellationToken.Token);
periodicBackup.RunningBackupTaskId = operationId;
periodicBackup.CancelToken = backupTask.TaskCancelToken;
var backupTaskName = $"{backupTypeText} backup task: '{periodicBackup.Configuration.Name}'";
var task = _database.Operations.AddOperation(
null,
backupTaskName,
Operations.Operations.OperationType.DatabaseBackup,
taskFactory: onProgress => Task.Run(async() =>
{
try
{
return(await backupTask.RunPeriodicBackup(onProgress));
}
finally
{
periodicBackup.RunningTask = null;
periodicBackup.RunningBackupTaskId = null;
periodicBackup.CancelToken = null;
periodicBackup.RunningBackupStatus = null;
if (periodicBackup.HasScheduledBackup() &&
_cancellationToken.IsCancellationRequested == false)
{
var newBackupTimer = GetTimer(periodicBackup.Configuration, periodicBackup.BackupStatus);
periodicBackup.UpdateTimer(newBackupTimer, discardIfDisabled: true);
}
}
}, backupTask.TaskCancelToken.Token),
id: operationId,
token: backupTask.TaskCancelToken);
periodicBackup.RunningTask = task;
task.ContinueWith(_ => backupTask.TaskCancelToken.Dispose());
return(operationId);
}
finally
{
periodicBackup.UpdateBackupTaskSemaphore.Release();
}
}
public async Task can_backup_and_restore()
{
var defaultS3Settings = GetS3Settings();
using (var store = GetDocumentStore())
{
using (var session = store.OpenAsyncSession())
{
await session.StoreAsync(new User { Name = "oren" }, "users/1");
session.CountersFor("users/1").Increment("likes", 100);
await session.SaveChangesAsync();
}
var config = new PeriodicBackupConfiguration
{
BackupType = BackupType.Backup,
S3Settings = defaultS3Settings,
IncrementalBackupFrequency = "0 0 1 1 *"
};
var backupTaskId = (await store.Maintenance.SendAsync(new UpdatePeriodicBackupOperation(config))).TaskId;
await store.Maintenance.SendAsync(new StartBackupOperation(true, backupTaskId));
var operation = new GetPeriodicBackupStatusOperation(backupTaskId);
PeriodicBackupStatus status = null;
var value = WaitForValue(() =>
{
status = store.Maintenance.Send(operation).Status;
return status?.LastEtag;
}, 4);
Assert.True(4 == value, $"4 == value, Got status: {status != null}, exception: {status?.Error?.Exception}");
Assert.True(status.LastOperationId != null, $"status.LastOperationId != null, Got status: {status != null}, exception: {status?.Error?.Exception}");
var backupOperation = store.Maintenance.Send(new GetOperationStateOperation(status.LastOperationId.Value));
var backupResult = backupOperation.Result as BackupResult;
Assert.True(backupResult != null && backupResult.Counters.Processed, "backupResult != null && backupResult.Counters.Processed");
Assert.True(1 == backupResult.Counters.ReadCount, "1 == backupResult.Counters.ReadCount");
using (var session = store.OpenAsyncSession())
{
await session.StoreAsync(new User { Name = "ayende" }, "users/2");
session.CountersFor("users/2").Increment("downloads", 200);
await session.SaveChangesAsync();
}
var lastEtag = store.Maintenance.Send(new GetStatisticsOperation()).LastDocEtag;
await store.Maintenance.SendAsync(new StartBackupOperation(false, backupTaskId));
value = WaitForValue(() => store.Maintenance.Send(operation).Status.LastEtag, lastEtag);
Assert.Equal(lastEtag, value);
// restore the database with a different name
var databaseName = $"restored_database-{Guid.NewGuid()}";
var subfolderS3Settings = GetS3Settings(status.FolderName);
using (RestoreDatabaseFromCloud(
store,
new RestoreFromS3Configuration { DatabaseName = databaseName, Settings = subfolderS3Settings, DisableOngoingTasks = true },
TimeSpan.FromSeconds(60)))
{
using (var session = store.OpenAsyncSession(databaseName))
{
var users = await session.LoadAsync<User>(new[] { "users/1", "users/2" });
Assert.True(users.Any(x => x.Value.Name == "oren"));
Assert.True(users.Any(x => x.Value.Name == "ayende"));
var val = await session.CountersFor("users/1").GetAsync("likes");
Assert.Equal(100, val);
val = await session.CountersFor("users/2").GetAsync("downloads");
Assert.Equal(200, val);
}
var originalDatabase = await Server.ServerStore.DatabasesLandlord.TryGetOrCreateResourceStore(store.Database);
var restoredDatabase = await Server.ServerStore.DatabasesLandlord.TryGetOrCreateResourceStore(databaseName);
using (restoredDatabase.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext ctx))
using (ctx.OpenReadTransaction())
{
var databaseChangeVector = DocumentsStorage.GetDatabaseChangeVector(ctx);
Assert.Equal($"A:7-{originalDatabase.DbBase64Id}, A:8-{restoredDatabase.DbBase64Id}", databaseChangeVector);
}
}
}
}
private static void ValidateBackupConfiguration(ServerWideBackupConfiguration serverWideConfiguration, PeriodicBackupConfiguration backupConfiguration, string databaseName)
{
Assert.Equal(PutServerWideBackupConfigurationCommand.GetTaskNameForDatabase(serverWideConfiguration.Name), backupConfiguration.Name);
Assert.Equal(serverWideConfiguration.Disabled, backupConfiguration.Disabled);
Assert.Equal(serverWideConfiguration.FullBackupFrequency, backupConfiguration.FullBackupFrequency);
Assert.Equal(serverWideConfiguration.IncrementalBackupFrequency, backupConfiguration.IncrementalBackupFrequency);
Assert.Equal($"{serverWideConfiguration.LocalSettings.FolderPath}{Path.DirectorySeparatorChar}{databaseName}", backupConfiguration.LocalSettings.FolderPath);
Assert.Equal(serverWideConfiguration.S3Settings.BucketName, backupConfiguration.S3Settings.BucketName);
Assert.Equal($"{serverWideConfiguration.S3Settings.RemoteFolderName}/{databaseName}", backupConfiguration.S3Settings.RemoteFolderName);
Assert.Equal(serverWideConfiguration.AzureSettings.AccountKey, backupConfiguration.AzureSettings.AccountKey);
Assert.Equal(serverWideConfiguration.AzureSettings.AccountName, backupConfiguration.AzureSettings.AccountName);
Assert.Equal($"{serverWideConfiguration.AzureSettings.RemoteFolderName}/{databaseName}", backupConfiguration.AzureSettings.RemoteFolderName);
Assert.Equal($"{serverWideConfiguration.FtpSettings.Url}/{databaseName}", backupConfiguration.FtpSettings.Url);
}
private void CreateUploadTaskIfNeeded <S, T>(
S settings,
List <Task> tasks,
string backupPath,
bool isFullBackup,
Func <S, FileStream, Progress, Task> uploadToServer,
T uploadStatus,
Action <IOperationProgress> onProgress)
where S : BackupSettings
where T : CloudUploadStatus
{
if (PeriodicBackupConfiguration.CanBackupUsing(settings) == false)
{
return;
}
Debug.Assert(uploadStatus != null);
var localUploadStatus = uploadStatus;
tasks.Add(Task.Run(async() =>
{
using (localUploadStatus.UpdateStats(isFullBackup))
using (var fileStream = File.OpenRead(backupPath))
{
var uploadProgress = localUploadStatus.UploadProgress;
localUploadStatus.Skipped = false;
uploadProgress.ChangeState(UploadState.PendingUpload);
uploadProgress.SetTotal(fileStream.Length);
AddInfo($"Starting {uploadStatus.GetType().AssemblyQualifiedName}", onProgress);
try
{
var bytesPutsPerSec = new MeterMetric();
long lastUploadedInBytes = 0;
var totalToUpload = new Sparrow.Size(uploadProgress.TotalInBytes, SizeUnit.Bytes).ToString();
var sw = Stopwatch.StartNew();
var progress = new Progress(uploadProgress)
{
OnUploadProgress = () =>
{
if (sw.ElapsedMilliseconds <= 1000)
{
return;
}
var totalUploadedInBytes = uploadProgress.UploadedInBytes;
bytesPutsPerSec.MarkSingleThreaded(totalUploadedInBytes - lastUploadedInBytes);
lastUploadedInBytes = totalUploadedInBytes;
var uploaded = new Sparrow.Size(totalUploadedInBytes, SizeUnit.Bytes);
uploadProgress.BytesPutsPerSec = bytesPutsPerSec.MeanRate;
AddInfo($"Uploaded: {uploaded} / {totalToUpload}", onProgress);
sw.Restart();
}
};
await uploadToServer(settings, fileStream, progress);
AddInfo($"Total uploaded: {totalToUpload}, " +
$"took: {MsToHumanReadableString(uploadProgress.UploadTimeInMs)}", onProgress);
}
catch (OperationCanceledException e)
{
// shutting down
localUploadStatus.Exception = e.ToString();
throw;
}
catch (Exception e)
{
localUploadStatus.Exception = e.ToString();
throw new InvalidOperationException($"Failed to backup to {uploadStatus.GetType().FullName}", e);
}
finally
{
uploadProgress.ChangeState(UploadState.Done);
}
}
}));
}
public async Task CanImportTombstonesFromIncrementalBackup()
{
var backupPath = NewDataPath(suffix: "BackupFolder");
using (var store = GetDocumentStore())
{
using (var session = store.OpenAsyncSession())
{
await session.StoreAsync(new User { Name = "fitzchak" }, "users/1");
await session.SaveChangesAsync();
}
var config = new PeriodicBackupConfiguration
{
LocalSettings = new LocalSettings
{
FolderPath = backupPath
},
IncrementalBackupFrequency = "* * * * *" //every minute
};
var result = await store.Admin.Server.SendAsync(new UpdatePeriodicBackupOperation(config, store.Database));
var periodicBackupTaskId = result.TaskId;
var operation = new GetPeriodicBackupStatusOperation(store.Database, periodicBackupTaskId);
SpinWait.SpinUntil(() =>
{
var getPeriodicBackupResult = store.Admin.Server.Send(operation);
return(getPeriodicBackupResult.Status?.LastEtag > 0);
}, TimeSpan.FromMinutes(2));
var etagForBackups = store.Admin.Server.Send(operation).Status.LastEtag;
using (var session = store.OpenAsyncSession())
{
session.Delete("users/1");
await session.SaveChangesAsync();
}
SpinWait.SpinUntil(() =>
{
var newLastEtag = store.Admin.Server.Send(operation).Status.LastEtag;
return(newLastEtag != etagForBackups);
}, TimeSpan.FromMinutes(2));
}
using (var store = GetDocumentStore(new Options
{
ModifyDatabaseName = s => $"{s}_2"
}))
{
await store.Smuggler.ImportIncrementalAsync(new DatabaseSmugglerImportOptions(),
Directory.GetDirectories(backupPath).First());
using (var session = store.OpenAsyncSession())
{
var user = await session.LoadAsync <User>("users/1");
Assert.Null(user);
}
}
}
public async Task AutoNamingAlgorithmOfOngoingTasksShouldTakeNameAlreadyExistsIntoAccount()
{
using (var store = GetDocumentStore())
{
var dbName = $"db/{Guid.NewGuid()}";
var csName = $"cs/{Guid.NewGuid()}";
var connectionString = new RavenConnectionString
{
Name = csName,
Database = dbName,
TopologyDiscoveryUrls = new[] { "http://127.0.0.1:12345" }
};
var result = await store.Maintenance.SendAsync(new PutConnectionStringOperation <RavenConnectionString>(connectionString));
Assert.NotNull(result.RaftCommandIndex);
await store.Maintenance.SendAsync(new UpdateExternalReplicationOperation(new ExternalReplication(dbName, csName)));
await store.Maintenance.SendAsync(new UpdateExternalReplicationOperation(new ExternalReplication(dbName, csName)));
var backupConfig = new PeriodicBackupConfiguration
{
LocalSettings = new LocalSettings
{
FolderPath = NewDataPath(suffix: "BackupFolder")
},
AzureSettings = new AzureSettings
{
StorageContainer = "abc"
},
FullBackupFrequency = "* */1 * * *",
IncrementalBackupFrequency = "* */2 * * *",
Disabled = true
};
await store.Maintenance.SendAsync(new UpdatePeriodicBackupOperation(backupConfig));
await store.Maintenance.SendAsync(new UpdatePeriodicBackupOperation(backupConfig));
var etlConfiguration = new RavenEtlConfiguration
{
ConnectionStringName = csName,
Transforms =
{
new Transformation()
{
Name = "loadAll",
Collections ={ "Users" },
Script = "loadToUsers(this)"
}
}
};
await store.Maintenance.SendAsync(new AddEtlOperation <RavenConnectionString>(etlConfiguration));
await store.Maintenance.SendAsync(new AddEtlOperation <RavenConnectionString>(etlConfiguration));
// for Pull Replication Hub name is required - no need to test
var sink = new PullReplicationAsSink
{
HubDefinitionName = "aa",
ConnectionString = connectionString,
ConnectionStringName = connectionString.Name
};
await store.Maintenance.SendAsync(new UpdatePullReplicationAsSinkOperation(sink));
await store.Maintenance.SendAsync(new UpdatePullReplicationAsSinkOperation(sink));
}
}
public async Task can_backup_and_restore_snapshot()
{
var backupPath = NewDataPath(suffix: "BackupFolder");
using (var store = GetDocumentStore())
{
using (var session = store.OpenAsyncSession())
{
await session.StoreAsync(new User { Name = "oren" }, "users/1");
await session.SaveChangesAsync();
}
var config = new PeriodicBackupConfiguration
{
BackupType = BackupType.Snapshot,
LocalSettings = new LocalSettings
{
FolderPath = backupPath
},
IncrementalBackupFrequency = "* * * * *" //every minute
};
var result = await store.Admin.Server.SendAsync(new UpdatePeriodicBackupOperation(config, store.Database));
var periodicBackupTaskId = result.TaskId;
var operation = new GetPeriodicBackupStatusOperation(store.Database, periodicBackupTaskId);
SpinWait.SpinUntil(() =>
{
var getPeriodicBackupResult = store.Admin.Server.Send(operation);
return(getPeriodicBackupResult.Status?.LastEtag > 0);
}, TimeSpan.FromMinutes(2));
var etagForBackups = store.Admin.Server.Send(operation).Status.LastEtag;
using (var session = store.OpenAsyncSession())
{
await session.StoreAsync(new User { Name = "ayende" }, "users/2");
await session.StoreAsync(new User { Name = "ayende" }, "users/3");
await session.SaveChangesAsync();
}
SpinWait.SpinUntil(() =>
{
var newLastEtag = store.Admin.Server.Send(operation).Status.LastEtag;
return(newLastEtag != etagForBackups);
}, TimeSpan.FromMinutes(2));
// restore the database with a different name
const string restoredDatabaseName = "restored_database_snapshot";
var restoreConfiguration = new RestoreBackupConfiguration
{
BackupLocation = Directory.GetDirectories(backupPath).First(),
DatabaseName = restoredDatabaseName
};
var restoreBackupTask = new RestoreBackupOperation(restoreConfiguration);
var restoreResult = store.Admin.Server.Send(restoreBackupTask);
var stateRequest = new GetOperationStateOperation(restoreResult.OperationId, true);
store.Admin.Server.Send(stateRequest);
SpinWait.SpinUntil(() =>
{
var state = store.Admin.Server.Send(stateRequest);
return(state.Status == OperationStatus.Completed);
}, TimeSpan.FromMinutes(2));
using (var session = store.OpenAsyncSession(restoredDatabaseName))
{
var users = await session.LoadAsync <User>(new[] { "users/1", "users/2" });
Assert.True(users.Any(x => x.Value.Name == "oren"));
Assert.True(users.Any(x => x.Value.Name == "ayende"));
}
}
}
public async Task can_backup_and_restore_snapshot()
{
var backupPath = NewDataPath(suffix: "BackupFolder");
using (var store = GetDocumentStore())
{
using (var session = store.OpenAsyncSession())
{
await session.StoreAsync(new User { Name = "oren" }, "users/1");
await session.SaveChangesAsync();
}
using (var session = store.OpenAsyncSession())
{
await session
.Query <User>()
.Where(x => x.Name == "oren")
.ToListAsync(); // create an index to backup
await session
.Query <User>()
.Where(x => x.Age > 20)
.ToListAsync(); // create an index to backup
session.CountersFor("users/1").Increment("likes", 100); //create a counter to backup
await session.SaveChangesAsync();
}
var config = new PeriodicBackupConfiguration
{
BackupType = BackupType.Snapshot,
LocalSettings = new LocalSettings
{
FolderPath = backupPath
},
IncrementalBackupFrequency = "* * * * *" //every minute
};
var backupTaskId = (await store.Maintenance.SendAsync(new UpdatePeriodicBackupOperation(config))).TaskId;
await store.Maintenance.SendAsync(new StartBackupOperation(true, backupTaskId));
var operation = new GetPeriodicBackupStatusOperation(backupTaskId);
SpinWait.SpinUntil(() =>
{
var getPeriodicBackupResult = store.Maintenance.Send(operation);
return(getPeriodicBackupResult.Status?.LastEtag > 0);
}, TimeSpan.FromSeconds(15));
var etagForBackups = store.Maintenance.Send(operation).Status.LastEtag;
using (var session = store.OpenAsyncSession())
{
await session.StoreAsync(new User { Name = "ayende" }, "users/2");
await session.StoreAsync(new User { Name = "ayende" }, "users/3");
session.CountersFor("users/2").Increment("downloads", 200);
await session.SaveChangesAsync();
}
await store.Maintenance.SendAsync(new StartBackupOperation(false, backupTaskId));
SpinWait.SpinUntil(() =>
{
var newLastEtag = store.Maintenance.Send(operation).Status.LastEtag;
return(newLastEtag != etagForBackups);
}, TimeSpan.FromSeconds(15));
// restore the database with a different name
string restoredDatabaseName = $"restored_database_snapshot-{Guid.NewGuid()}";
using (RestoreDatabase(store, new RestoreBackupConfiguration
{
BackupLocation = Directory.GetDirectories(backupPath).First(),
DatabaseName = restoredDatabaseName
}))
{
using (var session = store.OpenAsyncSession(restoredDatabaseName))
{
var users = await session.LoadAsync <User>(new[] { "users/1", "users/2" });
Assert.NotNull(users["users/1"]);
Assert.NotNull(users["users/2"]);
Assert.True(users.Any(x => x.Value.Name == "oren"));
Assert.True(users.Any(x => x.Value.Name == "ayende"));
var val = await session.CountersFor("users/1").GetAsync("likes");
Assert.Equal(100, val);
val = await session.CountersFor("users/2").GetAsync("downloads");
Assert.Equal(200, val);
}
var stats = await store.Maintenance.SendAsync(new GetStatisticsOperation());
Assert.Equal(2, stats.CountOfIndexes);
}
}
}
