public async Task CanDumpEmptyDatabase_Dumper()
{
var backupPath = NewDataPath("BackupFolder");
using (var server = GetNewServer())
{
using (new DocumentStore {
Url = "http://localhost:8079"
}.Initialize())
{
// now perform full backup
var options = new SmugglerOptions
{
BackupPath = backupPath,
};
var dumper = new DataDumper(server.Database, options);
var backupStatus = new PeriodicBackupStatus();
await dumper.ExportData(null, null, true, backupStatus);
}
}
VerifyDump(backupPath, store => Assert.Equal(0, store.DocumentDatabase.GetDocuments(0, int.MaxValue, null, CancellationToken.None).Count()));
IOExtensions.DeleteDirectory(backupPath);
}
/// <summary>
/// Run backup with provided task id and wait for completion. Full backup by default.
/// </summary>
/// <returns>PeriodicBackupStatus</returns>
public async Task <PeriodicBackupStatus> RunBackupAndReturnStatusAsync(RavenServer server, long taskId, DocumentStore store, bool isFullBackup = true, OperationStatus opStatus = OperationStatus.Completed, long?expectedEtag = default, int timeout = 15000)
{
await RunBackupAsync(server, taskId, store, isFullBackup, opStatus, timeout);
var operation = new GetPeriodicBackupStatusOperation(taskId);
PeriodicBackupStatus status = null;
if (expectedEtag.HasValue)
{
var etag = await WaitForValueAsync(async() =>
{
status = (await store.Maintenance.SendAsync(operation)).Status;
return(status.LastEtag);
}, expectedEtag.Value, interval : 333);
Assert.NotNull(etag);
Assert.Equal(expectedEtag.Value, etag.Value);
}
else
{
status = (await store.Maintenance.SendAsync(operation)).Status;
}
return(status);
}
public async Task CanPerformDump_Smuggler()
{
var backupPath = NewDataPath("BackupFolder");
using (var store = NewRemoteDocumentStore())
{
InsertUsers(store, 0, 2000);
var options = new SmugglerOptions
{
BackupPath = backupPath,
};
var dumper = new SmugglerApi(options, new RavenConnectionStringOptions
{
Url = "http://localhost:8079",
});
var backupStatus = new PeriodicBackupStatus();
await dumper.ExportData(null, null, true, backupStatus);
}
VerifyDump(backupPath, store =>
{
using (var session = store.OpenSession())
{
Assert.Equal(2000, session.Query <User>().Customize(x => x.WaitForNonStaleResultsAsOfNow()).Count());
}
});
IOExtensions.DeleteDirectory(backupPath);
}
public static void SaveBackupStatus(PeriodicBackupStatus status, DocumentDatabase documentDatabase, Logger logger, BackupResult backupResult)
{
try
{
var command = new UpdatePeriodicBackupStatusCommand(documentDatabase.Name, RaftIdGenerator.NewId())
{
PeriodicBackupStatus = status
};
var result = AsyncHelpers.RunSync(() => documentDatabase.ServerStore.SendToLeaderAsync(command));
AsyncHelpers.RunSync(() => documentDatabase.ServerStore.WaitForCommitIndexChange(RachisConsensus.CommitIndexModification.GreaterOrEqual, result.Index));
if (logger.IsInfoEnabled)
{
logger.Info($"Periodic backup status with task id {status.TaskId} was updated");
}
}
catch (Exception e)
{
const string message = "Error saving the periodic backup status";
if (logger.IsOperationsEnabled)
{
logger.Operations(message, e);
}
backupResult?.AddError($"{message}{Environment.NewLine}{e}");
}
}
private Timer GetTimer(
PeriodicBackupConfiguration configuration,
PeriodicBackupStatus backupStatus)
{
var nextBackup = GetNextBackupDetails(configuration, backupStatus);
if (nextBackup == null)
{
return(null);
}
if (_logger.IsInfoEnabled)
{
_logger.Info($"Next {(nextBackup.IsFull ? "full" : "incremental")} " +
$"backup is in {nextBackup.TimeSpan.TotalMinutes} minutes");
}
var backupTaskDetails = new BackupTaskDetails
{
IsFullBackup = nextBackup.IsFull,
TaskId = configuration.TaskId,
NextBackup = nextBackup.TimeSpan
};
var isValidTimeSpanForTimer = IsValidTimeSpanForTimer(backupTaskDetails.NextBackup);
var timer = isValidTimeSpanForTimer ?
new Timer(TimerCallback, backupTaskDetails, backupTaskDetails.NextBackup, Timeout.InfiniteTimeSpan) :
new Timer(LongPeriodTimerCallback, backupTaskDetails, MaxTimerTimeout, Timeout.InfiniteTimeSpan);
return(timer);
}
public BackupTask(DocumentDatabase database, BackupParameters backupParameters, BackupConfiguration configuration, Logger logger, PeriodicBackupRunner.TestingStuff forTestingPurposes = null)
{
_database = database;
_taskName = backupParameters.Name;
_operationId = backupParameters.OperationId;
_previousBackupStatus = backupParameters.BackupStatus;
_startTimeUtc = backupParameters.StartTimeUtc;
_isOneTimeBackup = backupParameters.IsOneTimeBackup;
_isFullBackup = backupParameters.IsFullBackup;
_backupToLocalFolder = backupParameters.BackupToLocalFolder;
_tempBackupPath = backupParameters.TempBackupPath;
_configuration = configuration;
_logger = logger;
_isServerWide = backupParameters.Name?.StartsWith(ServerWideBackupConfiguration.NamePrefix, StringComparison.OrdinalIgnoreCase) ?? false;
_isBackupEncrypted = IsBackupEncrypted(_database, _configuration);
_forTestingPurposes = forTestingPurposes;
_backupResult = GenerateBackupResult();
TaskCancelToken = new OperationCancelToken(_database.DatabaseShutdown, CancellationToken.None);
_retentionPolicyParameters = new RetentionPolicyBaseParameters
{
RetentionPolicy = backupParameters.RetentionPolicy,
DatabaseName = _database.Name,
IsFullBackup = _isFullBackup,
OnProgress = AddInfo,
CancellationToken = TaskCancelToken.Token
};
}
private async Task WriteStatus(PeriodicBackupStatus status, Action <IOperationProgress> onProgress)
{
AddInfo("Saving backup status", onProgress);
try
{
var command = new UpdatePeriodicBackupStatusCommand(_database.Name)
{
PeriodicBackupStatus = status
};
var result = await _serverStore.SendToLeaderAsync(command);
if (_logger.IsInfoEnabled)
{
_logger.Info($"Periodic backup status with task id {status.TaskId} was updated");
}
await _serverStore.WaitForCommitIndexChange(RachisConsensus.CommitIndexModification.GreaterOrEqual, result.Index);
}
catch (Exception e)
{
const string message = "Error saving the periodic backup status";
if (_logger.IsOperationsEnabled)
{
_logger.Operations(message, e);
}
}
}
private void WriteStatus(PeriodicBackupStatus status)
{
AddInfo("Saving backup status");
try
{
var command = new UpdatePeriodicBackupStatusCommand(_database.Name, RaftIdGenerator.NewId())
{
PeriodicBackupStatus = status
};
var result = AsyncHelpers.RunSync(() => _serverStore.SendToLeaderAsync(command));
if (_logger.IsInfoEnabled)
{
_logger.Info($"Periodic backup status with task id {status.TaskId} was updated");
}
AsyncHelpers.RunSync(() => _serverStore.WaitForCommitIndexChange(RachisConsensus.CommitIndexModification.GreaterOrEqual, result.Index));
}
catch (Exception e)
{
const string message = "Error saving the periodic backup status";
if (_logger.IsOperationsEnabled)
{
_logger.Operations(message, e);
}
}
}
public Task GetPeriodicBackupStatus()
{
var name = GetQueryStringValueAndAssertIfSingleAndNotEmpty("name");
if (TryGetAllowedDbs(name, out var _, requireAdmin: false) == false)
{
return(Task.CompletedTask);
}
var taskId = GetLongQueryString("taskId", required: true);
if (taskId == 0)
{
throw new ArgumentException("Task ID cannot be 0");
}
using (ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext context))
using (context.OpenReadTransaction())
using (var statusBlittable =
ServerStore.Cluster.Read(context, PeriodicBackupStatus.GenerateItemName(name, taskId.Value)))
using (var writer = new BlittableJsonTextWriter(context, ResponseBodyStream()))
{
writer.WriteStartObject();
writer.WritePropertyName(nameof(GetPeriodicBackupStatusOperationResult.Status));
writer.WriteObject(statusBlittable);
writer.WriteEndObject();
writer.Flush();
}
return(Task.CompletedTask);
}
public async Task CanDumpAttachmentsEmpty_Smuggler()
{
var backupPath = NewDataPath("BackupFolder");
using (NewRemoteDocumentStore())
{
var options = new SmugglerOptions
{
BackupPath = backupPath,
BatchSize = 100,
Limit = 206
};
var dumper = new SmugglerApi(options, new RavenConnectionStringOptions
{
Url = "http://localhost:8079",
});
var backupStatus = new PeriodicBackupStatus();
await dumper.ExportData(null, null, true, backupStatus);
}
VerifyDump(backupPath, store =>
{
Assert.Equal(0, store.DatabaseCommands.GetAttachmentHeadersStartingWith("user", 0, 500).Count());
});
IOExtensions.DeleteDirectory(backupPath);
}
private async Task <long> CreateBackup(DocumentStore store, bool isFullBackup, long backupTaskId, long lastEtag, int timeout)
{
await store.Maintenance.SendAsync(new StartBackupOperation(isFullBackup, backupTaskId));
var operation = new GetPeriodicBackupStatusOperation(backupTaskId);
PeriodicBackupStatus status = null;
var value = WaitForValue(() =>
{
status = store.Maintenance.Send(operation).Status;
if (status?.LastEtag == null)
{
return(false);
}
if (status.LastEtag.Value <= lastEtag)
{
return(false);
}
lastEtag = status.LastEtag.Value;
return(true);
}, true, timeout: timeout);
Assert.True(value, $"Got status: {status != null}, exception: {status?.Error?.Exception}, LocalBackup Exception: {status?.LocalBackup?.Exception}, lastEtag: {lastEtag}, status LastEtag: {status?.LastEtag}");
return(lastEtag);
}
public BackupTask(
ServerStore serverStore,
DocumentDatabase database,
PeriodicBackup periodicBackup,
bool isFullBackup,
bool backupToLocalFolder,
long operationId,
PathSetting tempBackupPath,
Logger logger,
CancellationToken databaseShutdownCancellationToken)
{
_serverStore = serverStore;
_database = database;
_startTime = periodicBackup.StartTime;
_periodicBackup = periodicBackup;
_configuration = periodicBackup.Configuration;
_previousBackupStatus = periodicBackup.BackupStatus;
_isFullBackup = isFullBackup;
_backupToLocalFolder = backupToLocalFolder;
_operationId = operationId;
_tempBackupPath = tempBackupPath;
_logger = logger;
_databaseShutdownCancellationToken = databaseShutdownCancellationToken;
TaskCancelToken = new OperationCancelToken(_databaseShutdownCancellationToken);
_backupResult = GenerateBackupResult();
}
private string PrintBackups(PeriodicBackupStatus pb, string[] dirs)
{
var sb = new StringBuilder();
sb.AppendLine($"LocalBackup:{Environment.NewLine}Directory:{pb.LocalBackup.BackupDirectory}, Exception: {pb.LocalBackup.Exception}");
sb.AppendLine($"Status:{Environment.NewLine}LastFullBackup: {pb.LastFullBackup}, FolderName: {pb.FolderName}, Exception:{pb.Error?.Exception}");
sb.AppendLine($"Dirs:{Environment.NewLine}{string.Join(", ", dirs)}");
return(sb.ToString());
}
public NextBackup GetNextBackupDetails(
DatabaseRecord databaseRecord,
PeriodicBackupConfiguration configuration,
PeriodicBackupStatus backupStatus)
{
var taskStatus = GetTaskStatus(databaseRecord, configuration, skipErrorLog: true);
return(taskStatus == TaskStatus.Disabled ? null : GetNextBackupDetails(configuration, backupStatus, skipErrorLog: true));
}
public NextBackup GetNextBackupDetails(
DatabaseRecord databaseRecord,
PeriodicBackupConfiguration configuration,
PeriodicBackupStatus backupStatus,
string responsibleNodeTag)
{
var taskStatus = GetTaskStatus(databaseRecord.Topology, configuration, skipErrorLog: true);
return(taskStatus == TaskStatus.Disabled ? null : GetNextBackupDetails(configuration, backupStatus, responsibleNodeTag, skipErrorLog: true));
}
public PeriodicBackupStatus GetBackupStatus(long taskId)
{
PeriodicBackupStatus inMemoryBackupStatus = null;
if (_periodicBackups.TryGetValue(taskId, out PeriodicBackup periodicBackup))
{
inMemoryBackupStatus = periodicBackup.BackupStatus;
}
return(GetBackupStatus(taskId, inMemoryBackupStatus));
}
public void FinishBackup(string backupName, PeriodicBackupStatus backupStatus, TimeSpan?elapsed, Logger logger)
{
lock (_locker)
{
_concurrentBackups++;
}
if (logger.IsOperationsEnabled)
{
string backupTypeString = "backup";
string extendedBackupTimings = string.Empty;
if (backupStatus != null)
{
backupTypeString = BackupTask.GetBackupDescription(backupStatus.BackupType, backupStatus.IsFull);
var first = true;
AddBackupTimings(backupStatus.LocalBackup, "local");
AddBackupTimings(backupStatus.UploadToS3, "Amazon S3");
AddBackupTimings(backupStatus.UploadToGlacier, "Amazon Glacier");
AddBackupTimings(backupStatus.UploadToAzure, "Azure");
AddBackupTimings(backupStatus.UploadToGoogleCloud, "Google Cloud");
AddBackupTimings(backupStatus.UploadToFtp, "FTP");
void AddBackupTimings(BackupStatus perDestinationBackupStatus, string backupTypeName)
{
if (perDestinationBackupStatus == null ||
perDestinationBackupStatus is CloudUploadStatus cus && cus.Skipped)
{
return;
}
if (first == false)
{
extendedBackupTimings += ", ";
}
first = false;
extendedBackupTimings +=
$"backup to {backupTypeName} took: " +
$"{(backupStatus.IsFull ? perDestinationBackupStatus.FullBackupDurationInMs : perDestinationBackupStatus.IncrementalBackupDurationInMs)}ms";
}
}
var message = $"Finished {backupTypeString} task '{backupName}'";
if (elapsed != null)
{
message += $", took: {elapsed}";
}
message += $" {extendedBackupTimings}";
logger.Operations(message);
}
}
private void ReadSetupValuesFromDocument()
{
using (LogContext.WithDatabase(Database.Name))
{
try
{
// Not having a setup doc means this DB isn't enabled for periodic backups
var document = Database.Get(PeriodicBackupSetup.RavenDocumentKey, null);
if (document == null)
{
backupConfigs = null;
backupStatus = null;
return;
}
var status = Database.Get(PeriodicBackupStatus.RavenDocumentKey, null);
backupStatus = status == null ? new PeriodicBackupStatus() : status.DataAsJson.JsonDeserialization <PeriodicBackupStatus>();
backupConfigs = document.DataAsJson.JsonDeserialization <PeriodicBackupSetup>();
if (backupConfigs.IntervalMilliseconds <= 0)
{
logger.Warn("Periodic backup interval is set to zero or less, periodic backup is now disabled");
return;
}
awsAccessKey = Database.Configuration.Settings["Raven/AWSAccessKey"];
awsSecretKey = Database.Configuration.Settings["Raven/AWSSecretKey"];
azureStorageAccount = Database.Configuration.Settings["Raven/AzureStorageAccount"];
azureStorageKey = Database.Configuration.Settings["Raven/AzureStorageKey"];
var interval = TimeSpan.FromMilliseconds(backupConfigs.IntervalMilliseconds);
logger.Info("Periodic backups started, will backup every" + interval.TotalMinutes + "minutes");
var timeSinceLastBackup = DateTime.UtcNow - backupStatus.LastBackup;
var nextBackup = timeSinceLastBackup >= interval ? TimeSpan.Zero : interval - timeSinceLastBackup;
timer = new Timer(TimerCallback, null, nextBackup, interval);
}
catch (Exception ex)
{
logger.ErrorException("Could not read periodic backup config", ex);
Database.AddAlert(new Alert
{
AlertLevel = AlertLevel.Error,
CreatedAt = SystemTime.UtcNow,
Message = ex.Message,
Title = "Could not read periodic backup config",
Exception = ex.ToString(),
UniqueKey = "Periodic Backup Config Error"
});
}
}
}
internal static PeriodicBackupStatus GetBackupStatusFromCluster(ServerStore serverStore, TransactionOperationContext context, string databaseName, long taskId)
{
var statusBlittable = serverStore.Cluster.Read(context, PeriodicBackupStatus.GenerateItemName(databaseName, taskId));
if (statusBlittable == null)
{
return(null);
}
var periodicBackupStatusJson = JsonDeserializationClient.PeriodicBackupStatus(statusBlittable);
return(periodicBackupStatusJson);
}
private void UpdateOperationId(PeriodicBackupStatus runningBackupStatus)
{
runningBackupStatus.LastOperationId = _operationId;
if (_previousBackupStatus.LastOperationId == null ||
_previousBackupStatus.NodeTag != _serverStore.NodeTag)
{
return;
}
// dismiss the previous operation
var id = $"{NotificationType.OperationChanged}/{_previousBackupStatus.LastOperationId.Value}";
_database.NotificationCenter.Dismiss(id);
}
private PeriodicBackupStatus GetBackupStatusFromCluster(long taskId)
{
using (_serverStore.ContextPool.AllocateOperationContext(out TransactionOperationContext context))
using (context.OpenReadTransaction())
{
var statusBlittable = _serverStore.Cluster.Read(context, PeriodicBackupStatus.GenerateItemName(_database.Name, taskId));
if (statusBlittable == null)
{
return(null);
}
var periodicBackupStatusJson = JsonDeserializationClient.PeriodicBackupStatus(statusBlittable);
return(periodicBackupStatusJson);
}
}
private NextBackup GetNextBackupDetails(
PeriodicBackupConfiguration configuration,
PeriodicBackupStatus backupStatus,
bool skipErrorLog = false)
{
var now = SystemTime.UtcNow;
var lastFullBackup = backupStatus.LastFullBackupInternal ?? now;
var lastIncrementalBackup = backupStatus.LastIncrementalBackupInternal ?? backupStatus.LastFullBackupInternal ?? now;
var nextFullBackup = GetNextBackupOccurrence(configuration.FullBackupFrequency,
lastFullBackup, configuration, skipErrorLog: skipErrorLog);
var nextIncrementalBackup = GetNextBackupOccurrence(configuration.IncrementalBackupFrequency,
lastIncrementalBackup, configuration, skipErrorLog: skipErrorLog);
if (nextFullBackup == null && nextIncrementalBackup == null)
{
var message = "Couldn't schedule next backup " +
$"full backup frequency: {configuration.FullBackupFrequency}, " +
$"incremental backup frequency: {configuration.IncrementalBackupFrequency}";
if (string.IsNullOrWhiteSpace(configuration.Name) == false)
{
message += $", backup name: {configuration.Name}";
}
_database.NotificationCenter.Add(AlertRaised.Create(
_database.Name,
"Couldn't schedule next backup, this shouldn't happen",
message,
AlertType.PeriodicBackup,
NotificationSeverity.Warning));
return(null);
}
Debug.Assert(configuration.TaskId != 0);
var isFullBackup = IsFullBackup(backupStatus, configuration, nextFullBackup, nextIncrementalBackup);
var nextBackupDateTime = GetNextBackupDateTime(nextFullBackup, nextIncrementalBackup);
var nowLocalTime = now.ToLocalTime();
var nextBackupTimeSpan = (nextBackupDateTime - nowLocalTime).Ticks <= 0 ? TimeSpan.Zero : nextBackupDateTime - nowLocalTime;
return(new NextBackup
{
TimeSpan = nextBackupTimeSpan,
DateTime = DateTime.UtcNow.Add(nextBackupTimeSpan),
IsFull = isFullBackup
});
}
public async Task CanPerformDumpWithLimitAndFilter_Smuggler()
{
var backupPath = NewDataPath("BackupFolder");
using (var store = NewRemoteDocumentStore())
{
var counter = 0;
counter = InsertUsers(store, counter, 1000);
counter = InsertDevelopers(store, counter, 2);
counter = InsertUsers(store, counter, 1000);
InsertDevelopers(store, counter, 2);
WaitForIndexing(store);
var options = new SmugglerOptions
{
Limit = 5,
BackupPath = backupPath,
Filters =
{
new FilterSetting
{
Path = "@metadata.Raven-Entity-Name",
Values = { "Developers" },
ShouldMatch = true,
}
}
};
var dumper = new SmugglerApi(options, new RavenConnectionStringOptions
{
Url = "http://localhost:8079",
});
var backupStatus = new PeriodicBackupStatus();
await dumper.ExportData(null, null, true, backupStatus);
}
VerifyDump(backupPath, store =>
{
using (var session = store.OpenSession())
{
Assert.Equal(4, session.Query <Developer>().Customize(x => x.WaitForNonStaleResultsAsOfNow()).Count());
}
});
IOExtensions.DeleteDirectory(backupPath);
}
private async Task RunBackupTask(long backupTaskId)
{
await DocumentStore.Maintenance.SendAsync(new StartBackupOperation(true, backupTaskId)).ConfigureAwait(false);
PeriodicBackupStatus backupStatus = DocumentStore.Maintenance.Send(new GetPeriodicBackupStatusOperation(backupTaskId)).Status;
var retries = 0;
while (backupStatus == null)
{
await Task.Delay(2000).ConfigureAwait(false);
backupStatus = DocumentStore.Maintenance.Send(new GetPeriodicBackupStatusOperation(backupTaskId)).Status;
if (++retries > 90) // 5 minutes
{
ReportFailure("RunBackupTask: Failed to get backup {backupTaskId} status", null);
return;
}
}
for (var i = 0; i < MyBackupsList.Count; i++)
{
if (MyBackupsList[i].BackupTaskId == backupTaskId)
{
OperationStatus operationStatus;
if (backupStatus.Error == null)
{
operationStatus = backupStatus.LastFullBackup == null ? OperationStatus.Canceled : OperationStatus.Completed;
}
else
{
operationStatus = OperationStatus.Faulted;
}
MyBackupsList[i].OperationStatus = operationStatus;
MyBackupsList[i].BackupStatus = backupStatus;
MyBackupsList[i].BackupPath = backupStatus.LocalBackup.BackupDirectory;
MyBackupsList[i].IsBackupCompleted = true;
break;
}
}
}
private async Task UploadToServer(
PeriodicBackupConfiguration configuration,
PeriodicBackupStatus backupStatus,
string backupPath, string folderName,
string fileName, bool isFullBackup)
{
if (_cancellationToken.IsCancellationRequested)
{
return;
}
var tasks = new List <Task>();
CreateUploadTaskIfNeeded(configuration.S3Settings, tasks, backupPath, isFullBackup,
async(settings, stream, uploadProgress) =>
{
var archiveDescription = GetArchiveDescription(isFullBackup, configuration.BackupType);
await UploadToS3(settings, stream, folderName, fileName, uploadProgress, archiveDescription);
},
ref backupStatus.UploadToS3, "S3");
CreateUploadTaskIfNeeded(configuration.GlacierSettings, tasks, backupPath, isFullBackup,
async(settings, stream, uploadProgress) =>
await UploadToGlacier(settings, stream, folderName, fileName, uploadProgress),
ref backupStatus.UploadToGlacier, "Glacier");
CreateUploadTaskIfNeeded(configuration.AzureSettings, tasks, backupPath, isFullBackup,
async(settings, stream, uploadProgress) =>
{
var archiveDescription = GetArchiveDescription(isFullBackup, configuration.BackupType);
await UploadToAzure(settings, stream, folderName, fileName, uploadProgress, archiveDescription);
},
ref backupStatus.UploadToAzure, "Azure");
CreateUploadTaskIfNeeded(configuration.FtpSettings, tasks, backupPath, isFullBackup,
async(settings, stream, uploadProgress) =>
await UploadToFtp(settings, stream, folderName, fileName, uploadProgress),
ref backupStatus.UploadToFtp, "FTP");
await Task.WhenAll(tasks);
}
public async Task CanDumpWhenHiddenDocsWithLimit_Smuggler()
{
var backupPath = NewDataPath("BackupFolder");
using (GetNewServer())
{
using (var store = new DocumentStore {
Url = "http://localhost:8079"
}.Initialize())
{
InsertHidenUsers(store, 2000);
var user1 = store.DatabaseCommands.Get("users/1");
Assert.Null(user1);
InsertUsers(store, 1, 25);
// now perform full backup
var options = new SmugglerOptions
{
BackupPath = backupPath,
};
var dumper = new SmugglerApi(options, new RavenConnectionStringOptions
{
Url = "http://localhost:8079",
});
var backupStatus = new PeriodicBackupStatus();
await dumper.ExportData(null, null, true, backupStatus);
}
}
VerifyDump(backupPath, store =>
{
using (var session = store.OpenSession())
{
Assert.Equal(25, session.Query <User>().Customize(x => x.WaitForNonStaleResultsAsOfNow()).Count());
}
});
IOExtensions.DeleteDirectory(backupPath);
}
public async Task CanDumpAttachments_Dumper()
{
var backupPath = NewDataPath("BackupFolder");
using (var store = NewDocumentStore())
{
InsertAttachments(store, 328);
var options = new SmugglerOptions
{
BackupPath = backupPath,
BatchSize = 100
};
var dumper = new DataDumper(store.DocumentDatabase, options);
var backupStatus = new PeriodicBackupStatus();
await dumper.ExportData(null, null, true, backupStatus);
}
VerifyDump(backupPath, store => Assert.Equal(328, store.DatabaseCommands.GetAttachmentHeadersStartingWith("user", 0, 500).Count()));
IOExtensions.DeleteDirectory(backupPath);
}
private PeriodicBackupStatus GetBackupStatus(long taskId, PeriodicBackupStatus inMemoryBackupStatus)
{
var backupStatus = GetBackupStatusFromCluster(taskId);
if (backupStatus == null)
{
backupStatus = inMemoryBackupStatus ?? new PeriodicBackupStatus
{
TaskId = taskId
};
}
else if (inMemoryBackupStatus?.Version > backupStatus.Version &&
inMemoryBackupStatus.NodeTag == backupStatus.NodeTag)
{
// the in memory backup status is more updated
// and is of the same node (current one)
backupStatus = inMemoryBackupStatus;
}
return(backupStatus);
}
public BackupTask(
ServerStore serverStore,
DocumentDatabase database,
PeriodicBackup periodicBackup,
bool isFullBackup,
bool backupToLocalFolder,
long operationId,
PathSetting tempBackupPath,
Logger logger,
CancellationToken databaseShutdownCancellationToken,
PeriodicBackupRunner.TestingStuff forTestingPurposes = null)
{
_serverStore = serverStore;
_database = database;
_periodicBackup = periodicBackup;
_configuration = periodicBackup.Configuration;
_isServerWide = _configuration.Name?.StartsWith(ServerWideBackupConfiguration.NamePrefix, StringComparison.OrdinalIgnoreCase) ?? false;
_isBackupEncrypted = IsBackupEncrypted(_database, _configuration);
_previousBackupStatus = periodicBackup.BackupStatus;
_isFullBackup = isFullBackup;
_backupToLocalFolder = backupToLocalFolder;
_operationId = operationId;
_tempBackupPath = tempBackupPath;
_logger = logger;
_databaseShutdownCancellationToken = databaseShutdownCancellationToken;
_forTestingPurposes = forTestingPurposes;
TaskCancelToken = new OperationCancelToken(_databaseShutdownCancellationToken);
_backupResult = GenerateBackupResult();
_retentionPolicyParameters = new RetentionPolicyBaseParameters
{
RetentionPolicy = _configuration.RetentionPolicy,
DatabaseName = _database.Name,
IsFullBackup = _isFullBackup,
OnProgress = AddInfo,
CancellationToken = TaskCancelToken.Token
};
}
private async Task WriteStatus(PeriodicBackupStatus status)
{
if (_cancellationToken.IsCancellationRequested)
{
return;
}
try
{
var command = new UpdatePeriodicBackupStatusCommand(_database.Name)
{
PeriodicBackupStatus = status
};
var result = await _serverStore.SendToLeaderAsync(command);
if (_logger.IsInfoEnabled)
{
_logger.Info($"Periodic backup status with task id {status.TaskId} was updated");
}
await _serverStore.WaitForCommitIndexChange(RachisConsensus.CommitIndexModification.GreaterOrEqual, result.Index);
}
catch (Exception e)
{
const string message = "Error saving the periodic backup status";
if (_logger.IsOperationsEnabled)
{
_logger.Operations(message, e);
}
_database.NotificationCenter.Add(AlertRaised.Create("Periodic Backup",
message,
AlertType.PeriodicBackup,
NotificationSeverity.Error,
details: new ExceptionDetails(e)));
}
}