public async Task ToggleDisableServerWideBackupFails()
{
using (var store = GetDocumentStore())
{
var putConfiguration = new ServerWideBackupConfiguration
{
Disabled = true,
FullBackupFrequency = "0 2 * * 0",
IncrementalBackupFrequency = "0 2 * * 1"
};
await store.Maintenance.Server.SendAsync(new PutServerWideBackupConfigurationOperation(putConfiguration));
var databaseRecord = await store.Maintenance.Server.SendAsync(new GetDatabaseRecordOperation(store.Database));
var currentBackupConfiguration = databaseRecord.PeriodicBackups.First();
var serverWideBackupTaskId = currentBackupConfiguration.TaskId;
var e = await Assert.ThrowsAsync <RavenException>(() => store.Maintenance.SendAsync(new ToggleOngoingTaskStateOperation(serverWideBackupTaskId, OngoingTaskType.Backup, false)));
Assert.Contains("Can't enable task name 'Server Wide Backup, Backup w/o destinations', because it is a server-wide backup task", e.Message);
e = await Assert.ThrowsAsync <RavenException>(() => store.Maintenance.SendAsync(new ToggleOngoingTaskStateOperation(serverWideBackupTaskId, OngoingTaskType.Backup, true)));
Assert.Contains("Can't disable task name 'Server Wide Backup, Backup w/o destinations', because it is a server-wide backup task", e.Message);
}
}
public async Task CanCreateMoreThanOneServerWideBackup()
{
using (var store = GetDocumentStore())
{
var putConfiguration = new ServerWideBackupConfiguration
{
Disabled = true,
FullBackupFrequency = "0 2 * * 0",
IncrementalBackupFrequency = "0 2 * * 1"
};
await store.Maintenance.Server.SendAsync(new PutServerWideBackupConfigurationOperation(putConfiguration));
putConfiguration.FtpSettings = new FtpSettings
{
Disabled = true,
Url = "http://url:8080"
};
await store.Maintenance.Server.SendAsync(new PutServerWideBackupConfigurationOperation(putConfiguration));
putConfiguration.AzureSettings = new AzureSettings
{
Disabled = true,
AccountKey = "test"
};
await store.Maintenance.Server.SendAsync(new PutServerWideBackupConfigurationOperation(putConfiguration));
var serverWideBackups = await store.Maintenance.Server.SendAsync(new GetServerWideBackupConfigurationsOperation());
Assert.Equal(3, serverWideBackups.Length);
var databaseRecord = await store.Maintenance.Server.SendAsync(new GetDatabaseRecordOperation(store.Database));
Assert.Equal(3, databaseRecord.PeriodicBackups.Count);
// update one of the tasks
var toUpdate = serverWideBackups[1];
toUpdate.BackupType = BackupType.Snapshot;
await store.Maintenance.Server.SendAsync(new PutServerWideBackupConfigurationOperation(toUpdate));
serverWideBackups = await store.Maintenance.Server.SendAsync(new GetServerWideBackupConfigurationsOperation());
Assert.Equal(3, serverWideBackups.Length);
databaseRecord = await store.Maintenance.Server.SendAsync(new GetDatabaseRecordOperation(store.Database));
Assert.Equal(3, databaseRecord.PeriodicBackups.Count);
// new database includes all server-wide backups
var newDbName = store.Database + "-testDatabase";
await store.Maintenance.Server.SendAsync(new CreateDatabaseOperation(new DatabaseRecord(newDbName)));
databaseRecord = await store.Maintenance.Server.SendAsync(new GetDatabaseRecordOperation(newDbName));
Assert.Equal(3, databaseRecord.PeriodicBackups.Count);
}
}
public async Task CanStoreAndEditServerWideBackupForIdleDatabase()
{
using var server = GetNewServer(new ServerCreationOptions
{
CustomSettings = new Dictionary <string, string>
{
[RavenConfiguration.GetKey(x => x.Databases.MaxIdleTime)] = "10",
[RavenConfiguration.GetKey(x => x.Databases.FrequencyToCheckForIdle)] = "3",
[RavenConfiguration.GetKey(x => x.Core.RunInMemory)] = "false"
}
});
using (var store = GetDocumentStore(new Options {
Server = server, RunInMemory = false
}))
{
var fullFreq = "0 2 1 1 *";
var incFreq = "0 2 * * 0";
var putConfiguration = new ServerWideBackupConfiguration
{
FullBackupFrequency = fullFreq,
IncrementalBackupFrequency = incFreq,
LocalSettings = new LocalSettings
{
FolderPath = "test/folder"
}
};
Assert.Equal(1, WaitForValue(() => server.ServerStore.IdleDatabases.Count, 1, timeout: 60000, interval: 1000));
var result = await store.Maintenance.Server.SendAsync(new PutServerWideBackupConfigurationOperation(putConfiguration));
var serverWideConfiguration = await store.Maintenance.Server.SendAsync(new GetServerWideBackupConfigurationOperation(result.Name));
Assert.NotNull(serverWideConfiguration);
Assert.Equal(fullFreq, serverWideConfiguration.FullBackupFrequency);
Assert.Equal(incFreq, serverWideConfiguration.IncrementalBackupFrequency);
Assert.Equal(1, server.ServerStore.IdleDatabases.Count);
// update the backup configuration
putConfiguration.FullBackupFrequency = "0 2 * * 0";
result = await store.Maintenance.Server.SendAsync(new PutServerWideBackupConfigurationOperation(putConfiguration));
serverWideConfiguration = await store.Maintenance.Server.SendAsync(new GetServerWideBackupConfigurationOperation(result.Name));
Assert.Equal(incFreq, serverWideConfiguration.FullBackupFrequency);
Assert.Equal(incFreq, serverWideConfiguration.IncrementalBackupFrequency);
Assert.Equal(1, server.ServerStore.IdleDatabases.Count);
}
}
private static void ValidateBackupConfiguration(ServerWideBackupConfiguration serverWideConfiguration, PeriodicBackupConfiguration backupConfiguration, string databaseName)
{
Assert.Equal(PutServerWideBackupConfigurationCommand.GetTaskNameForDatabase(serverWideConfiguration.Name), backupConfiguration.Name);
Assert.Equal(serverWideConfiguration.Disabled, backupConfiguration.Disabled);
Assert.Equal(serverWideConfiguration.FullBackupFrequency, backupConfiguration.FullBackupFrequency);
Assert.Equal(serverWideConfiguration.IncrementalBackupFrequency, backupConfiguration.IncrementalBackupFrequency);
Assert.Equal($"{serverWideConfiguration.LocalSettings.FolderPath}{Path.DirectorySeparatorChar}{databaseName}", backupConfiguration.LocalSettings.FolderPath);
Assert.Equal(serverWideConfiguration.S3Settings.BucketName, backupConfiguration.S3Settings.BucketName);
Assert.Equal($"{serverWideConfiguration.S3Settings.RemoteFolderName}/{databaseName}", backupConfiguration.S3Settings.RemoteFolderName);
Assert.Equal(serverWideConfiguration.AzureSettings.AccountKey, backupConfiguration.AzureSettings.AccountKey);
Assert.Equal(serverWideConfiguration.AzureSettings.AccountName, backupConfiguration.AzureSettings.AccountName);
Assert.Equal($"{serverWideConfiguration.AzureSettings.RemoteFolderName}/{databaseName}", backupConfiguration.AzureSettings.RemoteFolderName);
Assert.Equal($"{serverWideConfiguration.FtpSettings.Url}/{databaseName}", backupConfiguration.FtpSettings.Url);
}
private static void ValidateServerWideConfiguration(ServerWideBackupConfiguration serverWideConfiguration, ServerWideBackupConfiguration putConfiguration)
{
Assert.Equal(serverWideConfiguration.Name, putConfiguration.Name ?? putConfiguration.GetDefaultTaskName());
Assert.Equal(putConfiguration.Disabled, serverWideConfiguration.Disabled);
Assert.Equal(putConfiguration.FullBackupFrequency, serverWideConfiguration.FullBackupFrequency);
Assert.Equal(putConfiguration.IncrementalBackupFrequency, serverWideConfiguration.IncrementalBackupFrequency);
Assert.Equal(putConfiguration.LocalSettings.FolderPath, serverWideConfiguration.LocalSettings.FolderPath);
Assert.Equal(putConfiguration.S3Settings.BucketName, serverWideConfiguration.S3Settings.BucketName);
Assert.Equal(putConfiguration.S3Settings.RemoteFolderName, serverWideConfiguration.S3Settings.RemoteFolderName);
Assert.Equal(putConfiguration.AzureSettings.AccountKey, serverWideConfiguration.AzureSettings.AccountKey);
Assert.Equal(putConfiguration.AzureSettings.AccountName, serverWideConfiguration.AzureSettings.AccountName);
Assert.Equal(putConfiguration.AzureSettings.RemoteFolderName, serverWideConfiguration.AzureSettings.RemoteFolderName);
Assert.Equal(putConfiguration.FtpSettings.Url, serverWideConfiguration.FtpSettings.Url);
}
public async Task CreatePeriodicBackupFailsWhenUsingReservedName()
{
using (var store = GetDocumentStore())
{
var putConfiguration = new ServerWideBackupConfiguration
{
Disabled = true,
FullBackupFrequency = "0 2 * * 0",
IncrementalBackupFrequency = "0 2 * * 1"
};
await store.Maintenance.Server.SendAsync(new PutServerWideBackupConfigurationOperation(putConfiguration));
var databaseRecord = await store.Maintenance.Server.SendAsync(new GetDatabaseRecordOperation(store.Database));
var currentBackupConfiguration = databaseRecord.PeriodicBackups.First();
var serverWideBackupTaskId = currentBackupConfiguration.TaskId;
var backupConfiguration = new PeriodicBackupConfiguration
{
Disabled = true,
TaskId = currentBackupConfiguration.TaskId,
FullBackupFrequency = "0 2 * * 0",
IncrementalBackupFrequency = "0 2 * * 1"
};
var taskName = PutServerWideBackupConfigurationCommand.GetTaskNameForDatabase(putConfiguration.GetDefaultTaskName());
var e = await Assert.ThrowsAsync <RavenException>(() => store.Maintenance.SendAsync(new UpdatePeriodicBackupOperation(backupConfiguration)));
var expectedError = $"Can't delete task id: {currentBackupConfiguration.TaskId}, name: '{taskName}', because it is a server-wide backup task";
Assert.Contains(expectedError, e.Message);
backupConfiguration.TaskId = 0;
backupConfiguration.Name = currentBackupConfiguration.Name;
e = await Assert.ThrowsAsync <RavenException>(() => store.Maintenance.SendAsync(new UpdatePeriodicBackupOperation(backupConfiguration)));
expectedError = $"Can't create task: '{taskName}'. A regular (non server-wide) backup task name can't start with prefix '{ServerWideBackupConfiguration.NamePrefix}'";
Assert.Contains(expectedError, e.Message);
e = await Assert.ThrowsAsync <RavenException>(() => store.Maintenance.SendAsync(new DeleteOngoingTaskOperation(serverWideBackupTaskId, OngoingTaskType.Backup)));
expectedError = $"Can't delete task id: {serverWideBackupTaskId}, name: '{taskName}', because it is a server-wide backup task";
Assert.Contains(expectedError, e.Message);
}
}
public async Task CanCreateSnapshotBackupForNonEncryptedDatabase()
{
var backupPath = NewDataPath(suffix: "BackupFolder");
using (var store = GetDocumentStore())
{
var serverWideBackupConfiguration = new ServerWideBackupConfiguration
{
Disabled = false,
FullBackupFrequency = "0 2 * * 0",
IncrementalBackupFrequency = "0 2 * * 1",
BackupType = BackupType.Snapshot,
LocalSettings = new LocalSettings
{
FolderPath = backupPath
},
BackupEncryptionSettings = new BackupEncryptionSettings
{
Key = "OI7Vll7DroXdUORtc6Uo64wdAk1W0Db9ExXXgcg5IUs=",
EncryptionMode = EncryptionMode.UseProvidedKey
}
};
await store.Maintenance.Server.SendAsync(new PutServerWideBackupConfigurationOperation(serverWideBackupConfiguration));
var record = await store.Maintenance.Server.SendAsync(new GetDatabaseRecordOperation(store.Database));
var backup = record.PeriodicBackups.First();
var backupTaskId = backup.TaskId;
await store.Maintenance.SendAsync(new StartBackupOperation(true, backupTaskId));
var value = WaitForValue(() =>
{
var status = store.Maintenance.Send(new GetPeriodicBackupStatusOperation(backupTaskId)).Status;
return(status?.LastEtag);
}, 0);
Assert.Equal(0, value);
}
}
public async Task ToggleTaskState()
{
var disable = GetBoolValueQueryString("disable") ?? true;
var taskName = GetStringQueryString("taskName", required: false);
using (ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext context))
using (context.OpenReadTransaction())
{
// Get existing task
var serveWideBackupBlittable = ServerStore.Cluster.GetServerWideBackupConfigurations(context, taskName).FirstOrDefault();
if (serveWideBackupBlittable == null)
{
throw new InvalidOperationException($"Server-Wide Backup Task: {taskName} was not found in the server.");
}
// Toggle
ServerWideBackupConfiguration serverWideBackup = JsonDeserializationServer.ServerWideBackupConfiguration(serveWideBackupBlittable);
serverWideBackup.Disabled = disable;
// Save task
var(newIndex, _) = await ServerStore.PutServerWideBackupConfigurationAsync(serverWideBackup, GetRaftRequestIdFromQuery());
await ServerStore.WaitForCommitIndexChange(RachisConsensus.CommitIndexModification.GreaterOrEqual, newIndex);
using (var writer = new BlittableJsonTextWriter(context, ResponseBodyStream()))
{
var toggleResponse = new PutServerWideBackupConfigurationResponse()
{
Name = taskName,
RaftCommandIndex = newIndex
};
context.Write(writer, toggleResponse.ToJson());
writer.Flush();
}
}
}
public async Task SkipExportingTheServerWideBackup2()
{
var backupPath = NewDataPath(suffix: "BackupFolder");
using (var store = GetDocumentStore())
{
var serverWideBackupConfiguration = new ServerWideBackupConfiguration
{
Disabled = false,
FullBackupFrequency = "0 2 * * 0",
IncrementalBackupFrequency = "0 2 * * 1",
LocalSettings = new LocalSettings
{
FolderPath = backupPath
}
};
await store.Maintenance.Server.SendAsync(new PutServerWideBackupConfigurationOperation(serverWideBackupConfiguration));
var record = await store.Maintenance.Server.SendAsync(new GetDatabaseRecordOperation(store.Database));
var backup = record.PeriodicBackups.First();
var backupTaskId = backup.TaskId;
// save another backup task in the database record
var backupConfiguration = new PeriodicBackupConfiguration
{
Disabled = true,
FullBackupFrequency = "0 2 * * 0",
IncrementalBackupFrequency = "0 2 * * 1"
};
await store.Maintenance.SendAsync(new UpdatePeriodicBackupOperation(backupConfiguration));
await store.Maintenance.SendAsync(new StartBackupOperation(true, backupTaskId));
string backupDirectory = null;
var value = WaitForValue(() =>
{
var status = store.Maintenance.Send(new GetPeriodicBackupStatusOperation(backupTaskId)).Status;
backupDirectory = status?.LocalBackup.BackupDirectory;
return(status?.LastEtag);
}, 0);
Assert.Equal(0, value);
var files = Directory.GetFiles(backupDirectory)
.Where(BackupUtils.IsBackupFile)
.OrderBackups()
.ToArray();
var databaseName = GetDatabaseName() + "restore";
var restoreConfig = new RestoreBackupConfiguration
{
BackupLocation = backupDirectory,
DatabaseName = databaseName,
LastFileNameToRestore = files.OrderBackups().Last()
};
var restoreOperation = new RestoreBackupOperation(restoreConfig);
store.Maintenance.Server.Send(restoreOperation)
.WaitForCompletion(TimeSpan.FromSeconds(30));
// old server should have 2: 1 server-wide and 1 regular backup
using (var store2 = GetDocumentStore(new Options
{
CreateDatabase = false,
ModifyDatabaseName = s => databaseName,
}))
{
var record2 = await store.Maintenance.Server.SendAsync(new GetDatabaseRecordOperation(store2.Database));
Assert.Equal(2, record2.PeriodicBackups.Count);
}
// new server should have only one backup
var server = GetNewServer();
using (var store3 = GetDocumentStore(new Options
{
CreateDatabase = false,
ModifyDatabaseName = s => databaseName,
Server = server
}))
{
store3.Maintenance.Server.Send(restoreOperation)
.WaitForCompletion(TimeSpan.FromSeconds(30));
var record3 = await store3.Maintenance.Server.SendAsync(new GetDatabaseRecordOperation(databaseName));
Assert.Equal(1, record3.PeriodicBackups.Count);
}
}
}
public async Task ServerWideBackupShouldBeEncryptedForEncryptedDatabase(EncryptionMode encryptionMode)
{
var backupPath = NewDataPath(suffix: "BackupFolder");
var key = EncryptedServer(out var certificates, out string dbName);
using (var store = GetDocumentStore(new Options
{
AdminCertificate = certificates.ServerCertificate.Value,
ClientCertificate = certificates.ServerCertificate.Value,
ModifyDatabaseName = s => dbName,
ModifyDatabaseRecord = record => record.Encrypted = true,
Path = NewDataPath()
}))
{
var serverWideBackupConfiguration = new ServerWideBackupConfiguration
{
Disabled = false,
BackupType = BackupType.Backup,
LocalSettings = new LocalSettings
{
FolderPath = backupPath
},
IncrementalBackupFrequency = "0 */6 * * *"
};
switch (encryptionMode)
{
case EncryptionMode.None:
serverWideBackupConfiguration.BackupEncryptionSettings = new BackupEncryptionSettings
{
EncryptionMode = EncryptionMode.None,
};
break;
case EncryptionMode.UseDatabaseKey:
serverWideBackupConfiguration.BackupEncryptionSettings = new BackupEncryptionSettings
{
EncryptionMode = EncryptionMode.UseDatabaseKey
};
break;
case EncryptionMode.UseProvidedKey:
serverWideBackupConfiguration.BackupEncryptionSettings = new BackupEncryptionSettings
{
EncryptionMode = EncryptionMode.UseProvidedKey,
Key = "OI7Vll7DroXdUORtc6Uo64wdAk1W0Db9ExXXgcg5IUs="
};
break;
default:
throw new ArgumentOutOfRangeException(nameof(encryptionMode), encryptionMode, null);
}
await store.Maintenance.Server.SendAsync(new PutServerWideBackupConfigurationOperation(serverWideBackupConfiguration));
using (var session = store.OpenAsyncSession())
{
await session.StoreAsync(new User
{
Name = "grisha"
}, "users/1");
await session.SaveChangesAsync();
}
var record = await store.Maintenance.Server.SendAsync(new GetDatabaseRecordOperation(store.Database));
var backup = record.PeriodicBackups.First();
var backupTaskId = backup.TaskId;
await store.Maintenance.SendAsync(new StartBackupOperation(true, backupTaskId));
var value = WaitForValue(() =>
{
var status = store.Maintenance.Send(new GetPeriodicBackupStatusOperation(backupTaskId)).Status;
return(status?.LastEtag);
}, 1);
Assert.Equal(1, value);
var databaseName = $"restored_database-{Guid.NewGuid()}";
var backupDirectory = $"{backupPath}/{store.Database}";
using (RestoreDatabase(store, new RestoreBackupConfiguration
{
BackupLocation = Directory.GetDirectories(backupDirectory).First(),
DatabaseName = databaseName,
BackupEncryptionSettings = new BackupEncryptionSettings
{
EncryptionMode = EncryptionMode.UseProvidedKey,
Key = key
}
}))
{
using (var session = store.OpenSession(databaseName))
{
var users = session.Load <User>("users/1");
Assert.NotNull(users);
}
}
}
}
public async Task CanDeleteServerWideBackup()
{
using (var store = GetDocumentStore())
{
var putConfiguration = new ServerWideBackupConfiguration
{
Disabled = true,
FullBackupFrequency = "0 2 * * 0",
IncrementalBackupFrequency = "0 2 * * 1"
};
var result1 = await store.Maintenance.Server.SendAsync(new PutServerWideBackupConfigurationOperation(putConfiguration));
var result2 = await store.Maintenance.Server.SendAsync(new PutServerWideBackupConfigurationOperation(putConfiguration));
var record1 = await store.Maintenance.Server.SendAsync(new GetDatabaseRecordOperation(store.Database));
Assert.Equal(2, record1.PeriodicBackups.Count);
var serverWideBackups = await store.Maintenance.Server.SendAsync(new GetServerWideBackupConfigurationsOperation());
Assert.Equal(2, serverWideBackups.Length);
// the configuration is applied to new databases
var newDbName = store.Database + "-testDatabase";
await store.Maintenance.Server.SendAsync(new CreateDatabaseOperation(new DatabaseRecord(newDbName)));
var record2 = await store.Maintenance.Server.SendAsync(new GetDatabaseRecordOperation(newDbName));
Assert.Equal(2, record2.PeriodicBackups.Count);
await store.Maintenance.Server.SendAsync(new DeleteServerWideBackupConfigurationOperation(result1.Name));
var serverWideBackupConfiguration = await store.Maintenance.Server.SendAsync(new GetServerWideBackupConfigurationOperation(result1.Name));
Assert.Null(serverWideBackupConfiguration);
serverWideBackups = await store.Maintenance.Server.SendAsync(new GetServerWideBackupConfigurationsOperation());
Assert.Equal(1, serverWideBackups.Length);
// verify that the server-wide backup was deleted from all databases
record1 = await store.Maintenance.Server.SendAsync(new GetDatabaseRecordOperation(store.Database));
Assert.Equal(1, record1.PeriodicBackups.Count);
Assert.Equal($"{ServerWideBackupConfiguration.NamePrefix}, {putConfiguration.GetDefaultTaskName()} #2", record1.PeriodicBackups.First().Name);
record2 = await store.Maintenance.Server.SendAsync(new GetDatabaseRecordOperation(newDbName));
Assert.Equal(1, record2.PeriodicBackups.Count);
Assert.Equal($"{ServerWideBackupConfiguration.NamePrefix}, {putConfiguration.GetDefaultTaskName()} #2", record2.PeriodicBackups.First().Name);
await store.Maintenance.Server.SendAsync(new DeleteServerWideBackupConfigurationOperation(result2.Name));
serverWideBackupConfiguration = await store.Maintenance.Server.SendAsync(new GetServerWideBackupConfigurationOperation(result2.Name));
Assert.Null(serverWideBackupConfiguration);
serverWideBackups = await store.Maintenance.Server.SendAsync(new GetServerWideBackupConfigurationsOperation());
Assert.Equal(0, serverWideBackups.Length);
record1 = await store.Maintenance.Server.SendAsync(new GetDatabaseRecordOperation(store.Database));
Assert.Equal(0, record1.PeriodicBackups.Count);
record2 = await store.Maintenance.Server.SendAsync(new GetDatabaseRecordOperation(newDbName));
Assert.Equal(0, record2.PeriodicBackups.Count);
}
}
public async Task ServerWideBackupShouldBackupIdleDatabase(int rounds)
{
using var server = GetNewServer(new ServerCreationOptions
{
CustomSettings = new Dictionary <string, string>
{
[RavenConfiguration.GetKey(x => x.Databases.MaxIdleTime)] = "10",
[RavenConfiguration.GetKey(x => x.Databases.FrequencyToCheckForIdle)] = "3",
[RavenConfiguration.GetKey(x => x.Core.RunInMemory)] = "false"
}
});
var testDatabaseName = GetDatabaseName();
try
{
var backupPath = NewDataPath(suffix: "BackupFolder");
server.ServerStore.DatabasesLandlord.SkipShouldContinueDisposeCheck = true;
using var store = GetDocumentStore(new Options { Server = server, RunInMemory = false });
var dbName = store.Database;
store.Maintenance.Server.Send(new CreateDatabaseOperation(new DatabaseRecord(testDatabaseName)));
using (var session = store.OpenAsyncSession())
{
await session.StoreAsync(new User { Name = "EGOR" }, "su");
await session.SaveChangesAsync();
}
using (var session = store.OpenAsyncSession(testDatabaseName))
{
await session.StoreAsync(new User { Name = "egor" }, "susu");
await session.StoreAsync(new User { Name = "egor2" }, "sususu");
await session.SaveChangesAsync();
}
var first = true;
long backupTaskId = 0;
long opId1 = 0;
long opId2 = 0;
DateTime firstBackupStartTime = default;
for (int i = 0; i < rounds; i++)
{
// let db get idle
var now = DateTime.Now;
var nextNow = now + TimeSpan.FromSeconds(60);
while (now < nextNow && server.ServerStore.IdleDatabases.Count < 1)
{
await Task.Delay(3000);
await store.Maintenance.ForDatabase(testDatabaseName).SendAsync(new GetStatisticsOperation());
now = DateTime.Now;
}
Assert.True(1 == server.ServerStore.IdleDatabases.Count,
$"1 == server.ServerStore.IdleDatabases.Count({server.ServerStore.IdleDatabases.Count}), finishedOnTime? {now < nextNow}, now = {now}, nextNow = {nextNow}");
Assert.True(server.ServerStore.IdleDatabases.ContainsKey(dbName));
if (first)
{
firstBackupStartTime = DateTime.UtcNow;
var putConfiguration = new ServerWideBackupConfiguration
{
FullBackupFrequency = "*/2 * * * *",
LocalSettings = new LocalSettings {
FolderPath = backupPath
},
};
var result = await store.Maintenance.Server.SendAsync(new PutServerWideBackupConfigurationOperation(putConfiguration));
var serverWideConfiguration = await store.Maintenance.Server.SendAsync(new GetServerWideBackupConfigurationOperation(result.Name));
Assert.NotNull(serverWideConfiguration);
// the configuration is applied to existing databases
var record1 = await store.Maintenance.Server.SendAsync(new GetDatabaseRecordOperation(store.Database));
var backups1 = record1.PeriodicBackups;
Assert.Equal(1, backups1.Count);
backupTaskId = backups1.First().TaskId;
first = false;
}
//Wait for backup occurrence
nextNow = DateTime.Now + TimeSpan.FromSeconds(122);
while (now < nextNow && server.ServerStore.IdleDatabases.Count > 0)
{
await Task.Delay(2000);
store.Maintenance.ForDatabase(testDatabaseName).Send(new GetStatisticsOperation());
now = DateTime.Now;
}
Assert.True(0 == server.ServerStore.IdleDatabases.Count,
$"0 == server.ServerStore.IdleDatabases.Count({server.ServerStore.IdleDatabases.Count}), finishedOnTime? {now < nextNow}, now = {now}, nextNow = {nextNow}");
var operation = new GetPeriodicBackupStatusOperation(backupTaskId);
PeriodicBackupStatus status1 = null;
Assert.True(WaitForValue(() =>
{
status1 = store.Maintenance.ForDatabase(dbName).Send(operation).Status;
if (status1?.LastOperationId != null && status1.LastOperationId.Value > opId1)
{
opId1 = status1.LastOperationId.Value;
return(true);
}
return(false);
}, true));
PeriodicBackupStatus status2 = null;
Assert.True(WaitForValue(() =>
{
status2 = store.Maintenance.ForDatabase(testDatabaseName).Send(operation).Status;
if (status2?.LastOperationId != null && status2.LastOperationId.Value > opId2)
{
opId2 = status2.LastOperationId.Value;
return(true);
}
return(false);
}, true));
var backupsDir = Directory.GetDirectories(backupPath);
var backupsNum = backupsDir.Length;
var testDatabaseBackupPath = Path.Combine(backupPath, testDatabaseName);
var testDatabaseBackupDirs = Directory.GetDirectories(testDatabaseBackupPath);
var testDatabaseBackupNum = testDatabaseBackupDirs.Length;
var dbNameBackupPath = Path.Combine(backupPath, dbName);
var dbNameBackupDirs = Directory.GetDirectories(dbNameBackupPath);
var dbNameBackupNum = dbNameBackupDirs.Length;
Assert.Equal(2, backupsNum);
Assert.True(i + 1 == testDatabaseBackupNum,
$"firstBackupStartTime: {firstBackupStartTime}, i: {i}, testDatabaseBackupNum: {testDatabaseBackupNum}, path: {testDatabaseBackupPath}, {PrintBackups(status1, testDatabaseBackupDirs)}");
Assert.True(i + 1 == dbNameBackupNum,
$"firstBackupStartTime: {firstBackupStartTime}, i: {i}, dbNameBackupNum: {dbNameBackupNum}, path: {dbNameBackupPath}, {PrintBackups(status2, dbNameBackupDirs)}");
}
}
finally
{
server.ServerStore.DatabasesLandlord.SkipShouldContinueDisposeCheck = false;
}
}
public async Task CanStoreServerWideBackup()
{
using (var store = GetDocumentStore())
{
var putConfiguration = new ServerWideBackupConfiguration
{
Disabled = true,
FullBackupFrequency = "0 2 * * 0",
IncrementalBackupFrequency = "0 2 * * 1",
LocalSettings = new LocalSettings
{
FolderPath = "test/folder"
},
S3Settings = new S3Settings
{
BucketName = "ravendb-bucket",
RemoteFolderName = "grisha/backups"
},
AzureSettings = new AzureSettings
{
AccountKey = "Test",
AccountName = "Test",
RemoteFolderName = "grisha/backups"
},
FtpSettings = new FtpSettings
{
Url = "ftps://localhost/grisha/backups"
}
};
var result = await store.Maintenance.Server.SendAsync(new PutServerWideBackupConfigurationOperation(putConfiguration));
var serverWideConfiguration = await store.Maintenance.Server.SendAsync(new GetServerWideBackupConfigurationOperation(result.Name));
Assert.NotNull(serverWideConfiguration);
ValidateServerWideConfiguration(serverWideConfiguration, putConfiguration);
// the configuration is applied to existing databases
var record1 = await store.Maintenance.Server.SendAsync(new GetDatabaseRecordOperation(store.Database));
var backups1 = record1.PeriodicBackups;
Assert.Equal(1, backups1.Count);
ValidateBackupConfiguration(serverWideConfiguration, backups1.First(), store.Database);
// the configuration is applied to new databases
var newDbName = store.Database + "-testDatabase";
await store.Maintenance.Server.SendAsync(new CreateDatabaseOperation(new DatabaseRecord(newDbName)));
var backups2 = record1.PeriodicBackups;
Assert.Equal(1, backups2.Count);
var record2 = await store.Maintenance.Server.SendAsync(new GetDatabaseRecordOperation(newDbName));
ValidateBackupConfiguration(serverWideConfiguration, record2.PeriodicBackups.First(), newDbName);
// update the backup configuration
putConfiguration.FullBackupFrequency = "3 2 * * 1";
putConfiguration.LocalSettings.FolderPath += "/folder1";
putConfiguration.S3Settings.RemoteFolderName += "/folder2";
putConfiguration.AzureSettings.RemoteFolderName += "/folder3";
putConfiguration.FtpSettings.Url += "/folder4";
putConfiguration.Name = serverWideConfiguration.Name;
result = await store.Maintenance.Server.SendAsync(new PutServerWideBackupConfigurationOperation(putConfiguration));
serverWideConfiguration = await store.Maintenance.Server.SendAsync(new GetServerWideBackupConfigurationOperation(result.Name));
ValidateServerWideConfiguration(serverWideConfiguration, putConfiguration);
record1 = await store.Maintenance.Server.SendAsync(new GetDatabaseRecordOperation(store.Database));
Assert.Equal(1, record1.PeriodicBackups.Count);
ValidateBackupConfiguration(serverWideConfiguration, record1.PeriodicBackups.First(), store.Database);
record2 = await store.Maintenance.Server.SendAsync(new GetDatabaseRecordOperation(newDbName));
Assert.Equal(1, record2.PeriodicBackups.Count);
ValidateBackupConfiguration(serverWideConfiguration, record2.PeriodicBackups.First(), newDbName);
}
}
public async Task CanStoreAndEditServerWideBackupForIdleDatabase()
{
using var server = GetNewServer(new ServerCreationOptions
{
CustomSettings = new Dictionary <string, string>
{
[RavenConfiguration.GetKey(x => x.Databases.MaxIdleTime)] = "10",
[RavenConfiguration.GetKey(x => x.Databases.FrequencyToCheckForIdle)] = "3",
[RavenConfiguration.GetKey(x => x.Core.RunInMemory)] = "false"
}
});
using (var store = GetDocumentStore(new Options {
Server = server, RunInMemory = false
}))
{
var fullFreq = "0 2 1 1 *";
var incFreq = "0 2 * * 0";
var putConfiguration = new ServerWideBackupConfiguration
{
FullBackupFrequency = fullFreq,
IncrementalBackupFrequency = incFreq,
LocalSettings = new LocalSettings
{
FolderPath = "test/folder"
}
};
Assert.Equal(1, WaitForValue(() => server.ServerStore.IdleDatabases.Count, 1, timeout: 60000, interval: 1000));
var result = await store.Maintenance.Server.SendAsync(new PutServerWideBackupConfigurationOperation(putConfiguration));
var serverWideConfiguration = await store.Maintenance.Server.SendAsync(new GetServerWideBackupConfigurationOperation(result.Name));
Assert.NotNull(serverWideConfiguration);
Assert.Equal(fullFreq, serverWideConfiguration.FullBackupFrequency);
Assert.Equal(incFreq, serverWideConfiguration.IncrementalBackupFrequency);
Assert.Equal(1, server.ServerStore.IdleDatabases.Count);
// update the backup configuration
putConfiguration.Name = serverWideConfiguration.Name;
putConfiguration.TaskId = serverWideConfiguration.TaskId;
putConfiguration.FullBackupFrequency = "0 2 * * 0";
var oldName = result.Name;
result = await store.Maintenance.Server.SendAsync(new PutServerWideBackupConfigurationOperation(putConfiguration));
Exception ex = null;
try
{
await server.ServerStore.Cluster.WaitForIndexNotification(result.RaftCommandIndex, TimeSpan.FromMinutes(1));
}
catch (Exception e)
{
ex = e;
}
finally
{
Assert.Null(ex);
}
var record = await store.Maintenance.Server.SendAsync(new GetDatabaseRecordOperation(store.Database));
Assert.Equal(1, record.PeriodicBackups.Count);
PeriodicBackupConfiguration periodicBackupConfiguration = record.PeriodicBackups.First();
var newServerWideConfiguration = await store.Maintenance.Server.SendAsync(new GetServerWideBackupConfigurationOperation(result.Name));
// compare with periodic backup task
Assert.NotEqual(newServerWideConfiguration.TaskId, periodicBackupConfiguration.TaskId); // backup task id in db record doesn't change
Assert.Equal(PutServerWideBackupConfigurationCommand.GetTaskNameForDatabase(oldName), periodicBackupConfiguration.Name);
Assert.Equal(incFreq, periodicBackupConfiguration.FullBackupFrequency);
Assert.Equal(incFreq, periodicBackupConfiguration.IncrementalBackupFrequency);
Assert.NotEqual(serverWideConfiguration.FullBackupFrequency, periodicBackupConfiguration.FullBackupFrequency);
// compare with previous server wide backup
Assert.NotEqual(serverWideConfiguration.TaskId, newServerWideConfiguration.TaskId); // task id in server storage get increased with each change
Assert.Equal(oldName, result.Name);
Assert.Equal(incFreq, newServerWideConfiguration.FullBackupFrequency);
Assert.Equal(incFreq, newServerWideConfiguration.IncrementalBackupFrequency);
Assert.NotEqual(serverWideConfiguration.FullBackupFrequency, newServerWideConfiguration.FullBackupFrequency);
Assert.Equal(1, server.ServerStore.IdleDatabases.Count);
}
}
public async Task SkipExportingTheServerWideExternalReplication2()
{
var backupPath = NewDataPath(suffix: "BackupFolder");
using (var store = GetDocumentStore())
{
var putConfiguration1 = new ServerWideExternalReplication
{
Name = "1",
Disabled = true,
TopologyDiscoveryUrls = new[] { store.Urls.First() }
};
var putConfiguration2 = new ServerWideExternalReplication
{
Name = "2",
Disabled = true,
TopologyDiscoveryUrls = new[] { store.Urls.First() }
};
await store.Maintenance.Server.SendAsync(new PutServerWideExternalReplicationOperation(putConfiguration1));
await store.Maintenance.Server.SendAsync(new PutServerWideExternalReplicationOperation(putConfiguration2));
var dbName = $"db/{Guid.NewGuid()}";
var csName = $"cs/{Guid.NewGuid()}";
var connectionString = new RavenConnectionString
{
Name = csName,
Database = dbName,
TopologyDiscoveryUrls = new[] { "http://127.0.0.1:12345" }
};
var result = await store.Maintenance.SendAsync(new PutConnectionStringOperation <RavenConnectionString>(connectionString));
Assert.NotNull(result.RaftCommandIndex);
await store.Maintenance.SendAsync(new UpdateExternalReplicationOperation(new ExternalReplication(dbName, csName)
{
Disabled = true
}));
var databaseRecord = await store.Maintenance.Server.SendAsync(new GetDatabaseRecordOperation(store.Database));
Assert.Equal(3, databaseRecord.ExternalReplications.Count);
var serverWideBackupConfiguration = new ServerWideBackupConfiguration
{
Disabled = false,
FullBackupFrequency = "0 2 * * 0",
IncrementalBackupFrequency = "0 2 * * 1",
LocalSettings = new LocalSettings
{
FolderPath = backupPath
}
};
await store.Maintenance.Server.SendAsync(new PutServerWideBackupConfigurationOperation(serverWideBackupConfiguration));
var record = await store.Maintenance.Server.SendAsync(new GetDatabaseRecordOperation(store.Database));
var backup = record.PeriodicBackups.First();
var backupTaskId = backup.TaskId;
await store.Maintenance.SendAsync(new StartBackupOperation(true, backupTaskId));
string backupDirectory = null;
var value = WaitForValue(() =>
{
var status = store.Maintenance.Send(new GetPeriodicBackupStatusOperation(backupTaskId)).Status;
backupDirectory = status?.LocalBackup.BackupDirectory;
return(status?.LastEtag);
}, 0);
Assert.Equal(0, value);
var files = Directory.GetFiles(backupDirectory)
.Where(BackupUtils.IsBackupFile)
.OrderBackups()
.ToArray();
var databaseName = GetDatabaseName() + "restore";
var restoreConfig = new RestoreBackupConfiguration
{
BackupLocation = backupDirectory,
DatabaseName = databaseName,
LastFileNameToRestore = files.OrderBackups().Last()
};
var restoreOperation = new RestoreBackupOperation(restoreConfig);
store.Maintenance.Server.Send(restoreOperation)
.WaitForCompletion(TimeSpan.FromSeconds(30));
// new server should have only 0 external replications
var server = GetNewServer();
using (Databases.EnsureDatabaseDeletion(databaseName, store))
using (var store2 = GetDocumentStore(new Options
{
CreateDatabase = false,
ModifyDatabaseName = s => databaseName,
Server = server
}))
{
store2.Maintenance.Server.Send(restoreOperation)
.WaitForCompletion(TimeSpan.FromSeconds(30));
var record2 = await store2.Maintenance.Server.SendAsync(new GetDatabaseRecordOperation(databaseName));
Assert.Equal(1, record2.ExternalReplications.Count);
}
}
}
public async Task CanStoreServerWideBackup()
{
using (var store = GetDocumentStore())
{
#region server_wide_backup_configuration
var putConfiguration = new ServerWideBackupConfiguration
{
Disabled = true,
FullBackupFrequency = "0 2 * * 0",
IncrementalBackupFrequency = "0 2 * * 1",
//Backups are stored in this folder first, and sent from it to remote destinations (if defined).
LocalSettings = new LocalSettings
{
FolderPath = "localFolderPath"
},
//FTP settings
FtpSettings = new FtpSettings
{
Url = "ftps://localhost/john/backups"
},
//Microsoft Azure settings.
AzureSettings = new AzureSettings
{
AccountKey = "Azure Account Key",
AccountName = "Azure Account Name",
RemoteFolderName = "john/backups"
},
//Amazon S3 bucket settings.
S3Settings = new S3Settings
{
AwsAccessKey = "Amazon S3 Access Key",
AwsSecretKey = "Amazon S3 Secret Key",
AwsRegionName = "Amazon S3 Region Name",
BucketName = "john-bucket",
RemoteFolderName = "john/backups"
},
//Amazon Glacier settings.
GlacierSettings = new GlacierSettings
{
AwsAccessKey = "Amazon Glacier Access Key",
AwsSecretKey = "Amazon Glacier Secret Key",
AwsRegionName = "Amazon Glacier Region Name",
VaultName = "john-glacier",
RemoteFolderName = "john/backups"
},
//Google Cloud Backup settings
GoogleCloudSettings = new GoogleCloudSettings
{
BucketName = "Google Cloud Bucket",
RemoteFolderName = "BackupFolder",
GoogleCredentialsJson = "GoogleCredentialsJson"
}
};
var result = await store.Maintenance.Server.SendAsync(new PutServerWideBackupConfigurationOperation(putConfiguration));
var serverWideConfiguration = await store.Maintenance.Server.SendAsync(new GetServerWideBackupConfigurationOperation(result.Name));
#endregion
}
}
public async Task ServerWideBackupShouldNotWakeupIdleDatabases()
{
var backupPath = NewDataPath(suffix: "BackupFolder");
const int clusterSize = 3;
_databaseName = GetDatabaseName();
var leader = await CreateRaftClusterAndGetLeader(3, customSettings : new Dictionary <string, string>()
{
[RavenConfiguration.GetKey(x => x.Cluster.MoveToRehabGraceTime)] = "1",
[RavenConfiguration.GetKey(x => x.Cluster.AddReplicaTimeout)] = "1",
[RavenConfiguration.GetKey(x => x.Cluster.ElectionTimeout)] = "300",
[RavenConfiguration.GetKey(x => x.Cluster.StabilizationTime)] = "1",
[RavenConfiguration.GetKey(x => x.Databases.MaxIdleTime)] = "10",
[RavenConfiguration.GetKey(x => x.Databases.FrequencyToCheckForIdle)] = "3",
[RavenConfiguration.GetKey(x => x.Core.RunInMemory)] = "false"
});
DatabasePutResult databaseResult;
using (var store = new DocumentStore {
Urls = new[] { leader.WebUrl }, Database = _databaseName
}.Initialize())
{
var doc = new DatabaseRecord(_databaseName);
databaseResult = await store.Maintenance.Server.SendAsync(new CreateDatabaseOperation(doc, clusterSize));
}
Assert.Equal(clusterSize, databaseResult.Topology.AllNodes.Count());
foreach (var server in Servers)
{
await server.ServerStore.Cluster.WaitForIndexNotification(databaseResult.RaftCommandIndex);
}
foreach (var server in Servers.Where(s => databaseResult.NodesAddedTo.Any(n => n == s.WebUrl)))
{
await server.ServerStore.DatabasesLandlord.TryGetOrCreateResourceStore(_databaseName);
}
var rnd = new Random();
var index = rnd.Next(0, Servers.Count - 1);
using (var store = new DocumentStore {
Urls = new[] { Servers[index].WebUrl }, Database = _databaseName
}.Initialize())
{
using (var s = store.OpenAsyncSession())
{
await s.StoreAsync(new User()
{
Name = "Egor"
}, "foo/bar");
await s.SaveChangesAsync();
}
// wait until all dbs become idle
var now = DateTime.Now;
var nextNow = now + TimeSpan.FromSeconds(300);
while (now < nextNow && GetIdleCount() < 3)
{
await Task.Delay(3000);
now = DateTime.Now;
}
Assert.Equal(3, GetIdleCount());
var putConfiguration = new ServerWideBackupConfiguration
{
FullBackupFrequency = "*/1 * * * *",
IncrementalBackupFrequency = "*/1 * * * *",
LocalSettings = new LocalSettings {
FolderPath = backupPath
},
};
var result = await store.Maintenance.Server.SendAsync(new PutServerWideBackupConfigurationOperation(putConfiguration));
var serverWideConfiguration = await store.Maintenance.Server.SendAsync(new GetServerWideBackupConfigurationOperation(result.Name));
Assert.NotNull(serverWideConfiguration);
var record1 = await store.Maintenance.Server.SendAsync(new GetDatabaseRecordOperation(store.Database));
var backups1 = record1.PeriodicBackups;
_taskId = backups1.First().TaskId;
Assert.Equal(1, backups1.Count);
}
Assert.Equal(3, GetIdleCount());
// wait for the backup occurrence
var idleCount = WaitForCount(_reasonableWaitTime, 2, GetIdleCount);
Assert.Equal(2, idleCount);
// backup status should not wakeup dbs
var count = WaitForCount(_reasonableWaitTime, 3, CountOfBackupStatus);
Assert.Equal(3, count);
}
public async Task ServerWideBackupShouldBackupIdleDatabase(int rounds)
{
const string fullBackupFrequency = "*/2 * * * *";
var backupParser = CrontabSchedule.Parse(fullBackupFrequency);
const int maxIdleTimeInSec = 10;
using var server = GetNewServer(new ServerCreationOptions
{
CustomSettings = new Dictionary <string, string>
{
[RavenConfiguration.GetKey(x => x.Databases.MaxIdleTime)] = $"{maxIdleTimeInSec}",
[RavenConfiguration.GetKey(x => x.Databases.FrequencyToCheckForIdle)] = "3",
[RavenConfiguration.GetKey(x => x.Core.RunInMemory)] = "false"
}
});
using var dispose = new DisposableAction(() => server.ServerStore.DatabasesLandlord.SkipShouldContinueDisposeCheck = false);
server.ServerStore.DatabasesLandlord.SkipShouldContinueDisposeCheck = true;
var dbName = GetDatabaseName();
var controlgroupDbName = GetDatabaseName() + "controlgroup";
var baseBackupPath = NewDataPath(suffix: "BackupFolder");
using var store = new DocumentStore { Database = dbName, Urls = new[] { server.WebUrl } }.Initialize();
store.Maintenance.Server.Send(new CreateDatabaseOperation(new DatabaseRecord(dbName)));
store.Maintenance.Server.Send(new CreateDatabaseOperation(new DatabaseRecord(controlgroupDbName)));
await using var keepControlGroupAlive = new RepeatableAsyncAction(async token =>
{
await store.Maintenance.ForDatabase(controlgroupDbName).SendAsync(new GetStatisticsOperation(), token);
await Task.Delay(TimeSpan.FromSeconds(maxIdleTimeInSec), token);
}).Run();
using (var session = store.OpenAsyncSession())
{
await session.StoreAsync(new User { Name = "EGOR" }, "su");
await session.SaveChangesAsync();
}
using (var session = store.OpenAsyncSession(controlgroupDbName))
{
await session.StoreAsync(new User { Name = "egor" }, "susu");
await session.SaveChangesAsync();
}
var first = true;
DateTime backupStartTime = default;
GetPeriodicBackupStatusOperation periodicBackupStatusOperation = null;
PeriodicBackupStatus status = null;
PeriodicBackupStatus controlGroupStatus = null;
for (int i = 0; i < rounds; i++)
{
// let db get idle
await WaitForValueAsync(() => Task.FromResult(server.ServerStore.IdleDatabases.Count > 0), true, 180 * 1000, 3000);
Assert.True(1 == server.ServerStore.IdleDatabases.Count, $"IdleDatabasesCount({server.ServerStore.IdleDatabases.Count}), Round({i})");
Assert.True(server.ServerStore.IdleDatabases.ContainsKey(store.Database), $"Round({i})");
DateTime lastBackup;
if (first)
{
var putConfiguration = new ServerWideBackupConfiguration
{
FullBackupFrequency = fullBackupFrequency,
LocalSettings = new LocalSettings {
FolderPath = baseBackupPath
},
};
var result = await store.Maintenance.Server.SendAsync(new PutServerWideBackupConfigurationOperation(putConfiguration));
backupStartTime = DateTime.UtcNow;
var serverWideConfiguration = await store.Maintenance.Server.SendAsync(new GetServerWideBackupConfigurationOperation(result.Name));
Assert.NotNull(serverWideConfiguration);
periodicBackupStatusOperation = new GetPeriodicBackupStatusOperation(serverWideConfiguration.TaskId);
// the configuration is applied to existing databases
var periodicBackupConfigurations = (await store.Maintenance.Server.SendAsync(new GetDatabaseRecordOperation(store.Database))).PeriodicBackups;
Assert.Equal(1, periodicBackupConfigurations.Count);
var backupConfigurations = (await store.Maintenance.Server.SendAsync(new GetDatabaseRecordOperation(controlgroupDbName))).PeriodicBackups;
Assert.Equal(1, backupConfigurations.Count);
first = false;
lastBackup = backupStartTime;
}
else
{
Assert.True(status.LastFullBackup.HasValue);
lastBackup = status.LastFullBackup.Value;
}
var nextBackup = backupParser.GetNextOccurrence(lastBackup);
await Task.Delay(nextBackup - DateTime.UtcNow);
status = await AssertWaitForNextBackup(store.Database, status);
controlGroupStatus = await AssertWaitForNextBackup(controlgroupDbName, controlGroupStatus);
async Task <PeriodicBackupStatus> AssertWaitForNextBackup(string db, PeriodicBackupStatus prevStatus)
{
PeriodicBackupStatus nextStatus = null;
Assert.True(await WaitForValueAsync(async() =>
{
nextStatus = (await store.Maintenance.ForDatabase(db).SendAsync(periodicBackupStatusOperation)).Status;
if (nextStatus == null)
{
return(false);
}
Assert.True(nextStatus.Error?.Exception == null, nextStatus.Error?.Exception);
Assert.True(nextStatus.LocalBackup?.Exception == null, nextStatus.LocalBackup?.Exception);
return(prevStatus == null || nextStatus.LastOperationId.HasValue && nextStatus.LastOperationId > prevStatus.LastOperationId);
}, true), $"Round {i}");
return(nextStatus);
}
var backupsDir = Directory.GetDirectories(baseBackupPath);
Assert.Equal(2, backupsDir.Length);
AssertBackupDirCount(controlgroupDbName, controlGroupStatus);
AssertBackupDirCount(store.Database, status);
void AssertBackupDirCount(string db, PeriodicBackupStatus periodicBackupStatus)
{
var backupPath = Path.Combine(baseBackupPath, db);
var backupDirs = Directory.GetDirectories(backupPath);
Assert.True(i + 1 == backupDirs.Length,
$"firstBackupStartTime: {backupStartTime}, checkTime: {DateTime.UtcNow}, i: {i}, " +
$"controlGroupBackupNum: {backupDirs.Length}, path: {backupPath}, {PrintBackups(periodicBackupStatus, backupDirs)}");
}
}
}
public async Task CanCreateBackupUsingConfigurationFromBackupScript()
{
var backupPath = NewDataPath(suffix: "BackupFolder");
using (var store = GetDocumentStore())
{
var scriptPath = Path.Combine(Path.GetTempPath(), Path.ChangeExtension(Guid.NewGuid().ToString(), ".ps1"));
var localSetting = new LocalSettings
{
Disabled = false,
FolderPath = backupPath,
};
var localSettingsString = JsonConvert.SerializeObject(localSetting);
string command;
string script;
if (PlatformDetails.RunningOnPosix)
{
command = "bash";
script = $"#!/bin/bash\r\necho '{localSettingsString}'";
File.WriteAllText(scriptPath, script);
Process.Start("chmod", $"700 {scriptPath}");
}
else
{
command = "powershell";
script = $"echo '{localSettingsString}'";
File.WriteAllText(scriptPath, script);
}
var putConfiguration = new ServerWideBackupConfiguration
{
Disabled = false,
FullBackupFrequency = "0 2 * * 0",
IncrementalBackupFrequency = "0 2 * * 1",
LocalSettings = new LocalSettings
{
GetBackupConfigurationScript = new GetBackupConfigurationScript
{
Exec = command,
Arguments = scriptPath
}
}
};
await store.Maintenance.Server.SendAsync(new PutServerWideBackupConfigurationOperation(putConfiguration));
var record = await store.Maintenance.Server.SendAsync(new GetDatabaseRecordOperation(store.Database));
var backupTask = record.PeriodicBackups.First();
Assert.Null(backupTask.LocalSettings.FolderPath);
Assert.NotNull(backupTask.LocalSettings.GetBackupConfigurationScript);
Assert.NotNull(backupTask.LocalSettings.GetBackupConfigurationScript.Exec);
Assert.NotNull(backupTask.LocalSettings.GetBackupConfigurationScript.Arguments);
var backupTaskId = backupTask.TaskId;
using (var session = store.OpenAsyncSession())
{
await session.StoreAsync(new Company { Name = "Hibernating Rhinos" }, "companies/1");
await session.SaveChangesAsync();
}
await store.Maintenance.SendAsync(new StartBackupOperation(true, backupTaskId));
var operation = new GetPeriodicBackupStatusOperation(backupTaskId);
var value = WaitForValue(() =>
{
var status = store.Maintenance.Send(operation).Status;
return(status?.LastEtag);
}, 1);
Assert.Equal(1, value);
}
}
public async Task ServerWideBackupShouldBackupIdleDatabase(int rounds)
{
using var server = GetNewServer(new ServerCreationOptions
{
CustomSettings = new Dictionary <string, string>
{
[RavenConfiguration.GetKey(x => x.Databases.MaxIdleTime)] = "10",
[RavenConfiguration.GetKey(x => x.Databases.FrequencyToCheckForIdle)] = "3",
[RavenConfiguration.GetKey(x => x.Core.RunInMemory)] = "false"
}
});
try
{
var backupPath = NewDataPath(suffix: "BackupFolder");
server.ServerStore.DatabasesLandlord.SkipShouldContinueDisposeCheck = true;
using var store = GetDocumentStore(new Options { Server = server, RunInMemory = false });
var dbName = store.Database;
store.Maintenance.Server.Send(new CreateDatabaseOperation(new DatabaseRecord("Test")));
using (var session = store.OpenAsyncSession())
{
await session.StoreAsync(new User { Name = "EGOR" }, "su");
await session.SaveChangesAsync();
}
using (var session = store.OpenAsyncSession("Test"))
{
await session.StoreAsync(new User { Name = "egor" }, "susu");
await session.StoreAsync(new User { Name = "egor2" }, "sususu");
await session.SaveChangesAsync();
}
var first = true;
long backupTaskId = 0;
for (int i = 0; i < rounds; i++)
{
// let db get idle
var now = DateTime.Now;
var nextNow = now + TimeSpan.FromSeconds(60);
while (now < nextNow && server.ServerStore.IdleDatabases.Count < 1)
{
Thread.Sleep(3000);
await store.Maintenance.ForDatabase("Test").SendAsync(new GetStatisticsOperation());
now = DateTime.Now;
}
Assert.True(1 == server.ServerStore.IdleDatabases.Count,
$"1 == server.ServerStore.IdleDatabases.Count({server.ServerStore.IdleDatabases.Count}), finishedOnTime? {now < nextNow}, now = {now}, nextNow = {nextNow}");
if (first)
{
var putConfiguration = new ServerWideBackupConfiguration
{
FullBackupFrequency = "*/2 * * * *",
LocalSettings = new LocalSettings {
FolderPath = backupPath
},
};
var result = await store.Maintenance.Server.SendAsync(new PutServerWideBackupConfigurationOperation(putConfiguration));
var serverWideConfiguration = await store.Maintenance.Server.SendAsync(new GetServerWideBackupConfigurationOperation(result.Name));
Assert.NotNull(serverWideConfiguration);
// the configuration is applied to existing databases
var record1 = await store.Maintenance.Server.SendAsync(new GetDatabaseRecordOperation(store.Database));
var backups1 = record1.PeriodicBackups;
Assert.Equal(1, backups1.Count);
backupTaskId = backups1.First().TaskId;
first = false;
}
//Wait for backup occurrence
nextNow = DateTime.Now + TimeSpan.FromSeconds(122);
while (now < nextNow && server.ServerStore.IdleDatabases.Count > 0)
{
Thread.Sleep(2000);
store.Maintenance.ForDatabase("Test").Send(new GetStatisticsOperation());
now = DateTime.Now;
}
Assert.True(0 == server.ServerStore.IdleDatabases.Count,
$"0 == server.ServerStore.IdleDatabases.Count({server.ServerStore.IdleDatabases.Count}), finishedOnTime? {now < nextNow}, now = {now}, nextNow = {nextNow}");
var operation = new GetPeriodicBackupStatusOperation(backupTaskId);
var value = WaitForValue(() =>
{
var status = store.Maintenance.Send(operation).Status;
return(status?.LastEtag);
}, 1);
Assert.Equal(1, value);
Assert.True(2 == Directory.GetDirectories(backupPath).Length, $"2 == Directory.GetDirectories(backupPath).Length({Directory.GetDirectories(backupPath).Length})");
Assert.True(i + 1 == Directory.GetDirectories(Path.Combine(backupPath, "Test")).Length, $"i + 1 == Directory.GetDirectories(Path.Combine(backupPath, 'Test')).Length({Directory.GetDirectories(Path.Combine(backupPath, "Test")).Length})");
Assert.True(i + 1 == Directory.GetDirectories(Path.Combine(backupPath, dbName)).Length, $"i + 1 == Directory.GetDirectories(Path.Combine(backupPath, dbName)).Length({Directory.GetDirectories(Path.Combine(backupPath, dbName)).Length})");
}
}
finally
{
server.ServerStore.DatabasesLandlord.SkipShouldContinueDisposeCheck = false;
}
}
public async Task ServerWideBackupShouldNotWakeupIdleDatabases()
{
var backupPath = NewDataPath(suffix: "BackupFolder");
const int clusterSize = 3;
_databaseName = GetDatabaseName();
var cluster = await CreateRaftCluster(numberOfNodes : clusterSize, shouldRunInMemory : false, customSettings : new Dictionary <string, string>()
{
[RavenConfiguration.GetKey(x => x.Cluster.MoveToRehabGraceTime)] = "10",
[RavenConfiguration.GetKey(x => x.Cluster.AddReplicaTimeout)] = "1",
[RavenConfiguration.GetKey(x => x.Cluster.ElectionTimeout)] = "300",
[RavenConfiguration.GetKey(x => x.Cluster.StabilizationTime)] = "1",
[RavenConfiguration.GetKey(x => x.Databases.MaxIdleTime)] = "10",
[RavenConfiguration.GetKey(x => x.Databases.FrequencyToCheckForIdle)] = "3"
});
_nodes = cluster.Nodes;
try
{
foreach (var server in _nodes)
{
server.ServerStore.DatabasesLandlord.SkipShouldContinueDisposeCheck = true;
}
using (var store = GetDocumentStore(new Options
{
ModifyDatabaseName = s => _databaseName,
ReplicationFactor = clusterSize,
Server = cluster.Leader,
RunInMemory = false
}))
{
using (var s = store.OpenAsyncSession())
{
await s.StoreAsync(new User()
{
Name = "Egor"
}, "foo/bar");
await s.SaveChangesAsync();
}
var idleCount = WaitForCount(_reasonableWaitTime, 3, GetIdleCount);
Assert.Equal(3, idleCount);
var putConfiguration = new ServerWideBackupConfiguration
{
FullBackupFrequency = "*/1 * * * *",
IncrementalBackupFrequency = "*/1 * * * *",
LocalSettings = new LocalSettings {
FolderPath = backupPath
},
};
var result = await store.Maintenance.Server.SendAsync(new PutServerWideBackupConfigurationOperation(putConfiguration));
var serverWideConfiguration = await store.Maintenance.Server.SendAsync(new GetServerWideBackupConfigurationOperation(result.Name));
Assert.NotNull(serverWideConfiguration);
var record1 = await store.Maintenance.Server.SendAsync(new GetDatabaseRecordOperation(store.Database));
var backups1 = record1.PeriodicBackups;
_taskId = backups1.First().TaskId;
Assert.Equal(_taskId, serverWideConfiguration.TaskId);
Assert.Equal(1, backups1.Count);
Assert.Equal(3, GetIdleCount());
// wait for the backup occurrence
idleCount = WaitForCount(_reasonableWaitTime, 2, GetIdleCount);
Assert.Equal(2, idleCount);
var reasons = new Dictionary <string, string>();
// backup status should not wakeup dbs
var count = WaitForCount(_reasonableWaitTime, 3, () => CountOfBackupStatus(out reasons));
var sb = new StringBuilder();
foreach (var kvp in reasons)
{
sb.AppendLine($"Node {kvp.Key}, backup status:{Environment.NewLine}{kvp.Value}");
sb.AppendLine();
}
Assert.True(3 == count, $"3 == count{Environment.NewLine}{sb.ToString()}");
}
}
finally
{
foreach (var server in _nodes)
{
server.ServerStore.DatabasesLandlord.SkipShouldContinueDisposeCheck = false;
}
}
}
