public void MaxNumberOfItemsToProcessInSingleBatchShouldBeRespectedByDataDumper()
{
var path = Path.Combine(NewDataPath(forceCreateDir: true), "raven.dump");
using (var server = GetNewServer(configureConfig: configuration => configuration.MaxNumberOfItemsToProcessInSingleBatch = 1234))
{
var dumper = new DataDumper(server.SystemDatabase, options: new SmugglerOptions { BatchSize = 4321 });
Assert.Equal(4321, dumper.SmugglerOptions.BatchSize);
dumper.ExportData(new SmugglerExportOptions { ToFile = path }).ResultUnwrap();
Assert.Equal(1234, dumper.SmugglerOptions.BatchSize);
dumper = new DataDumper(server.SystemDatabase, options: new SmugglerOptions { BatchSize = 4321 });
Assert.Equal(4321, dumper.SmugglerOptions.BatchSize);
dumper.ImportData(new SmugglerImportOptions { FromFile = path }).Wait();
Assert.Equal(1234, dumper.SmugglerOptions.BatchSize);
dumper = new DataDumper(server.SystemDatabase, options: new SmugglerOptions { BatchSize = 1000 });
Assert.Equal(1000, dumper.SmugglerOptions.BatchSize);
dumper.ExportData(new SmugglerExportOptions { ToFile = path }).ResultUnwrap();
Assert.Equal(1000, dumper.SmugglerOptions.BatchSize);
}
}
public async Task CanPerformDump_Dumper()
{
var backupPath = NewDataPath("BackupFolder");
using (var store = NewDocumentStore())
{
InsertUsers(store, 0, 2000);
var dumper = new DataDumper(store.DocumentDatabase);
await dumper.ExportData(new SmugglerExportOptions
{
ToFile = backupPath,
}, new SmugglerOptions
{
Incremental = true
});
}
VerifyDump(backupPath, store =>
{
using (var session = store.OpenSession())
{
Assert.Equal(2000, session.Query<User>().Customize(x => x.WaitForNonStaleResultsAsOfNow()).Count());
}
});
IOExtensions.DeleteDirectory(backupPath);
}
private void TimerCallback(object state)
{
if (currentTask != null)
return;
lock (this)
{
if (currentTask != null)
return;
currentTask = Task.Factory.StartNew(() =>
{
var documentDatabase = Database;
if (documentDatabase == null)
return;
using (LogContext.WithDatabase(documentDatabase.Name))
{
try
{
var localBackupConfigs = backupConfigs;
var localBackupStatus = backupStatus;
if (localBackupConfigs == null)
return;
var databaseStatistics = documentDatabase.Statistics;
// No-op if nothing has changed
if (databaseStatistics.LastDocEtag == localBackupStatus.LastDocsEtag &&
databaseStatistics.LastAttachmentEtag == localBackupStatus.LastAttachmentsEtag)
{
return;
}
var backupPath = localBackupConfigs.LocalFolderName ??
Path.Combine(documentDatabase.Configuration.DataDirectory, "PeriodicBackup-Temp");
var options = new SmugglerOptions
{
BackupPath = backupPath,
LastDocsEtag = localBackupStatus.LastDocsEtag,
LastAttachmentEtag = localBackupStatus.LastAttachmentsEtag
};
var dd = new DataDumper(documentDatabase, options);
var filePath = dd.ExportData(null, true);
// No-op if nothing has changed
if (options.LastDocsEtag == localBackupStatus.LastDocsEtag &&
options.LastAttachmentEtag == localBackupStatus.LastAttachmentsEtag)
{
logger.Info("Periodic backup returned prematurely, nothing has changed since last backup");
return;
}
UploadToServer(filePath, localBackupConfigs);
localBackupStatus.LastAttachmentsEtag = options.LastAttachmentEtag;
localBackupStatus.LastDocsEtag = options.LastDocsEtag;
var ravenJObject = RavenJObject.FromObject(localBackupStatus);
ravenJObject.Remove("Id");
var putResult = documentDatabase.Put(PeriodicBackupStatus.RavenDocumentKey, null, ravenJObject,
new RavenJObject(), null);
// this result in backupStatus being refreshed
localBackupStatus = backupStatus;
if (localBackupStatus != null)
{
if (Etag.Increment(localBackupStatus.LastDocsEtag, 1) == putResult.ETag) // the last etag is with just us
localBackupStatus.LastDocsEtag = putResult.ETag; // so we can skip it for the next time
}
}
catch (ObjectDisposedException)
{
// shutting down, probably
}
catch (Exception e)
{
logger.ErrorException("Error when performing periodic backup", e);
Database.AddAlert(new Alert
{
AlertLevel = AlertLevel.Error,
CreatedAt = SystemTime.UtcNow,
Message = e.Message,
Title = "Error in Periodic Backup",
Exception = e.ToString(),
UniqueKey = "Periodic Backup Error",
});
}
}
})
.ContinueWith(_ =>
{
currentTask = null;
});
}
}
private void TimerCallback(bool fullBackup)
{
if (currentTask != null)
return;
// we have shared lock for both incremental and full backup.
lock (this)
{
if (currentTask != null)
return;
currentTask = Task.Factory.StartNew(async () =>
{
var documentDatabase = Database;
if (documentDatabase == null)
return;
using (LogContext.WithDatabase(documentDatabase.Name))
{
try
{
var dataDumper = new DataDumper(documentDatabase);
var localBackupConfigs = exportConfigs;
var localBackupStatus = exportStatus;
if (localBackupConfigs == null)
return;
if (fullBackup == false)
{
var currentEtags = dataDumper.Operations.FetchCurrentMaxEtags();
// No-op if nothing has changed
if (currentEtags.LastDocsEtag == localBackupStatus.LastDocsEtag &&
currentEtags.LastAttachmentsEtag == localBackupStatus.LastAttachmentsEtag &&
currentEtags.LastDocDeleteEtag == localBackupStatus.LastDocsDeletionEtag &&
currentEtags.LastAttachmentsDeleteEtag == localBackupStatus.LastAttachmentDeletionEtag)
{
return;
}
}
var backupPath = localBackupConfigs.LocalFolderName ??
Path.Combine(documentDatabase.Configuration.DataDirectory, "PeriodicExport-Temp");
if (fullBackup)
{
// create filename for full dump
backupPath = Path.Combine(backupPath, SystemTime.UtcNow.ToString("yyyy-MM-dd-HH-mm", CultureInfo.InvariantCulture) + ".ravendb-full-dump");
if (File.Exists(backupPath))
{
var counter = 1;
while (true)
{
backupPath = Path.Combine(Path.GetDirectoryName(backupPath), SystemTime.UtcNow.ToString("yyyy-MM-dd-HH-mm", CultureInfo.InvariantCulture) + " - " + counter + ".ravendb-full-dump");
if (File.Exists(backupPath) == false)
break;
counter++;
}
}
}
var smugglerOptions = dataDumper.SmugglerOptions;
if (fullBackup == false)
{
smugglerOptions.StartDocsEtag = localBackupStatus.LastDocsEtag;
smugglerOptions.StartAttachmentsEtag = localBackupStatus.LastAttachmentsEtag;
smugglerOptions.StartDocsDeletionEtag = localBackupStatus.LastDocsDeletionEtag;
smugglerOptions.StartAttachmentsDeletionEtag = localBackupStatus.LastAttachmentDeletionEtag;
smugglerOptions.Incremental = true;
smugglerOptions.ExportDeletions = true;
}
var exportResult = await dataDumper.ExportData(new SmugglerExportOptions { ToFile = backupPath });
if (fullBackup == false)
{
// No-op if nothing has changed
if (exportResult.LastDocsEtag == localBackupStatus.LastDocsEtag &&
exportResult.LastAttachmentsEtag == localBackupStatus.LastAttachmentsEtag &&
exportResult.LastDocDeleteEtag == localBackupStatus.LastDocsDeletionEtag &&
exportResult.LastAttachmentsDeleteEtag == localBackupStatus.LastAttachmentDeletionEtag)
{
logger.Info("Periodic export returned prematurely, nothing has changed since last export");
return;
}
}
try
{
if (!localBackupConfigs.Disabled)
{
UploadToServer(exportResult.FilePath, localBackupConfigs, fullBackup);
}
}
finally
{
// if user did not specify local folder we delete temporary file.
if (String.IsNullOrEmpty(localBackupConfigs.LocalFolderName))
{
IOExtensions.DeleteFile(exportResult.FilePath);
}
}
if (fullBackup)
{
localBackupStatus.LastFullBackup = SystemTime.UtcNow;
}
else
{
localBackupStatus.LastAttachmentsEtag = exportResult.LastAttachmentsEtag;
localBackupStatus.LastDocsEtag = exportResult.LastDocsEtag;
localBackupStatus.LastDocsDeletionEtag = exportResult.LastDocDeleteEtag;
localBackupStatus.LastAttachmentDeletionEtag = exportResult.LastAttachmentsDeleteEtag;
localBackupStatus.LastBackup = SystemTime.UtcNow;
}
var ravenJObject = JsonExtensions.ToJObject(localBackupStatus);
ravenJObject.Remove("Id");
var putResult = documentDatabase.Documents.Put(PeriodicExportStatus.RavenDocumentKey, null, ravenJObject,
new RavenJObject(), null);
// this result in exportStatus being refreshed
localBackupStatus = exportStatus;
if (localBackupStatus != null)
{
if (localBackupStatus.LastDocsEtag.IncrementBy(1) == putResult.ETag) // the last etag is with just us
localBackupStatus.LastDocsEtag = putResult.ETag; // so we can skip it for the next time
}
}
catch (ObjectDisposedException)
{
// shutting down, probably
}
catch (Exception e)
{
logger.ErrorException("Error when performing periodic export", e);
Database.AddAlert(new Alert
{
AlertLevel = AlertLevel.Error,
CreatedAt = SystemTime.UtcNow,
Message = e.Message,
Title = "Error in Periodic Export",
Exception = e.ToString(),
UniqueKey = "Periodic Export Error",
});
}
}
})
.Unwrap();
currentTask.ContinueWith(_ =>
{
currentTask = null;
});
}
}
private void TimerCallback(object state)
{
if (executing)
return;
executing = true;
PeriodicBackupSetup backupConfigs;
try
{
// Setup doc might be deleted or changed by the user
var document = Database.Get(PeriodicBackupSetup.RavenDocumentKey, null);
if (document == null)
{
timer.Dispose();
timer = null;
return;
}
backupConfigs = document.DataAsJson.JsonDeserialization<PeriodicBackupSetup>();
if (backupConfigs.Interval <= 0)
{
timer.Dispose();
timer = null;
return;
}
}
catch (Exception ex)
{
logger.WarnException(ex.Message, ex);
executing = false;
return;
}
try
{
var options = new SmugglerOptions
{
BackupPath = Path.GetTempPath(), //TODO temp path in data folder instead
LastDocsEtag = backupConfigs.LastDocsEtag,
LastAttachmentEtag = backupConfigs.LastAttachmentsEtag
};
var dd = new DataDumper(Database, options);
var filePath = dd.ExportData(null, true);
// No-op if nothing has changed
if (options.LastDocsEtag == backupConfigs.LastDocsEtag && options.LastAttachmentEtag == backupConfigs.LastAttachmentsEtag)
{
logger.Info("Periodic backup returned prematurely, nothing has changed since last backup");
return;
}
DoUpload(filePath, backupConfigs);
// Remember the current position only once we are successful, this allows for compensatory backups
// in case of failures. We reload the setup document to make sure we don't override changes made by
// the user.
// Setup doc might be deleted or changed by the user
var document = Database.Get(PeriodicBackupSetup.RavenDocumentKey, null);
if (document == null)
{
timer.Dispose();
timer = null;
return;
}
backupConfigs = document.DataAsJson.JsonDeserialization<PeriodicBackupSetup>();
backupConfigs.LastAttachmentsEtag = options.LastAttachmentEtag;
backupConfigs.LastDocsEtag = options.LastDocsEtag;
Database.Put(PeriodicBackupSetup.RavenDocumentKey, null, RavenJObject.FromObject(backupConfigs),
new RavenJObject(), null);
if (backupConfigs.Interval != interval)
{
if (backupConfigs.Interval <= 0)
{
timer.Dispose();
timer = null;
}
else
{
interval = backupConfigs.Interval;
timer.Change(TimeSpan.FromMinutes(backupConfigs.Interval), TimeSpan.FromMinutes(backupConfigs.Interval));
}
}
}
catch (Exception e)
{
logger.ErrorException("Error when performing periodic backup", e);
}
finally
{
executing = false;
}
}
private void TimerCallback(object state)
{
if (currentTask != null)
return;
lock (this)
{
if (currentTask != null)
return;
currentTask = Task.Factory.StartNew(() =>
{
using (LogManager.OpenMappedContext("database", Database.Name ?? Constants.SystemDatabase))
using (new DisposableAction(() => LogContext.DatabaseName.Value = null))
{
LogContext.DatabaseName.Value = Database.Name;
try
{
var localBackupConfigs = backupConfigs;
if (localBackupConfigs == null)
return;
var backupPath = localBackupConfigs.LocalFolderName ??
Path.Combine(Database.Configuration.DataDirectory, "PeriodicBackup-Temp");
var options = new SmugglerOptions
{
BackupPath = backupPath,
LastDocsEtag = localBackupConfigs.LastDocsEtag,
LastAttachmentEtag = localBackupConfigs.LastAttachmentsEtag
};
var dd = new DataDumper(Database, options);
var filePath = dd.ExportData(null, true);
// No-op if nothing has changed
if (options.LastDocsEtag == backupConfigs.LastDocsEtag &&
options.LastAttachmentEtag == backupConfigs.LastAttachmentsEtag)
{
logger.Info("Periodic backup returned prematurely, nothing has changed since last backup");
return;
}
UploadToServer(filePath, localBackupConfigs);
localBackupConfigs.LastAttachmentsEtag = options.LastAttachmentEtag;
localBackupConfigs.LastDocsEtag = options.LastDocsEtag;
if (backupConfigs == null) // it was removed by the user?
{
localBackupConfigs.IntervalMilliseconds = -1; // this disable the periodic backup
}
var ravenJObject = RavenJObject.FromObject(localBackupConfigs);
ravenJObject.Remove("Id");
var putResult = Database.Put(PeriodicBackupSetup.RavenDocumentKey, null, ravenJObject,
new RavenJObject(), null);
if (Etag.Increment(localBackupConfigs.LastDocsEtag, 1) == putResult.ETag) // the last etag is with just us
localBackupConfigs.LastDocsEtag = putResult.ETag; // so we can skip it for the next time
}
catch (ObjectDisposedException)
{
// shutting down, probably
}
catch (Exception e)
{
Database.AddAlert(new Alert
{
AlertLevel = AlertLevel.Error,
CreatedAt = SystemTime.UtcNow,
Message = e.Message,
Title = "Error in Periodic Backup",
Exception = e
});
logger.ErrorException("Error when performing periodic backup", e);
}
}
})
.ContinueWith(_ =>
{
currentTask = null;
});
}
}
public async Task CanDumpAttachmentsEmpty_Dumper()
{
var backupPath = NewDataPath("BackupFolder");
using (var store = NewDocumentStore())
{
var options = new SmugglerOptions
{
BackupPath = backupPath,
BatchSize = 100,
Limit = 206
};
var dumper = new DataDumper(store.DocumentDatabase, options);
var backupStatus = new PeriodicBackupStatus();
await dumper.ExportData(null, null, true, backupStatus);
}
VerifyDump(backupPath, store =>
{
Assert.Equal(0, store.DatabaseCommands.GetAttachmentHeadersStartingWith("user", 0, 500).Count());
});
IOExtensions.DeleteDirectory(backupPath);
}
public async Task CanDumpWhenHiddenDocsWithLimit_Dumper()
{
var backupPath = NewDataPath("BackupFolder");
using (var server = GetNewServer())
{
using (var store = new DocumentStore { Url = "http://localhost:8079" }.Initialize())
{
InsertHidenUsers(store, 2000);
var user1 = store.DatabaseCommands.Get("users/1");
Assert.Null(user1);
InsertUsers(store, 1, 25);
// now perform full backup
var options = new SmugglerOptions
{
BackupPath = backupPath,
};
var dumper = new DataDumper(server.Database, options);
var backupStatus = new PeriodicBackupStatus();
await dumper.ExportData(null, null, true, backupStatus);
}
}
VerifyDump(backupPath, store =>
{
using (var session = store.OpenSession())
{
Assert.Equal(25, session.Query<User>().Customize(x => x.WaitForNonStaleResultsAsOfNow()).Count());
}
});
IOExtensions.DeleteDirectory(backupPath);
}
public async Task CanDumpEmptyDatabase_Dumper()
{
var backupPath = NewDataPath("BackupFolder");
using (var server = GetNewServer())
{
using (new DocumentStore { Url = "http://localhost:8079" }.Initialize())
{
// now perform full backup
var options = new SmugglerOptions
{
BackupPath = backupPath,
};
var dumper = new DataDumper(server.Database, options);
var backupStatus = new PeriodicBackupStatus();
await dumper.ExportData(null, null, true, backupStatus);
}
}
VerifyDump(backupPath, store => Assert.Equal(0, store.DocumentDatabase.GetDocuments(0, int.MaxValue, null, CancellationToken.None).Count()));
IOExtensions.DeleteDirectory(backupPath);
}
public async Task CanPerformDumpWithLimitAndFilter_Dumper()
{
var backupPath = NewDataPath("BackupFolder");
using (var store = NewDocumentStore())
{
var counter = 0;
counter = InsertUsers(store, counter, 1000);
counter = InsertDevelopers(store, counter, 2);
counter = InsertUsers(store, counter, 1000);
InsertDevelopers(store, counter, 2);
WaitForIndexing(store);
var options = new SmugglerOptions
{
Limit = 5,
BackupPath = backupPath,
Filters =
{
new FilterSetting
{
Path = "@metadata.Raven-Entity-Name",
Values = {"Developers"},
ShouldMatch = true,
}
}
};
var dumper = new DataDumper(store.DocumentDatabase, options);
var backupStatus = new PeriodicBackupStatus();
await dumper.ExportData(null, null, true, backupStatus);
}
VerifyDump(backupPath, store =>
{
using (var session = store.OpenSession())
{
Assert.Equal(4, session.Query<Developer>().Customize(x => x.WaitForNonStaleResultsAsOfNow()).Count());
}
});
IOExtensions.DeleteDirectory(backupPath);
}
public Task<HttpResponseMessage> ExportDatabase(ExportData smugglerOptionsJson)
{
var requestString = smugglerOptionsJson.SmugglerOptions;
SmugglerOptions smugglerOptions;
using (var jsonReader = new RavenJsonTextReader(new StringReader(requestString)))
{
var serializer = JsonExtensions.CreateDefaultJsonSerializer();
smugglerOptions = (SmugglerOptions)serializer.Deserialize(jsonReader, typeof(SmugglerOptions));
}
var result = GetEmptyMessage();
// create PushStreamContent object that will be called when the output stream will be ready.
result.Content = new PushStreamContent(async (outputStream, content, arg3) =>
{
try
{
var dataDumper = new DataDumper(Database, smugglerOptions);
await dataDumper.ExportData(
new SmugglerExportOptions
{
ToStream = outputStream
}).ConfigureAwait(false);
}
finally
{
outputStream.Close();
}
});
result.Content.Headers.ContentDisposition = new ContentDispositionHeaderValue("attachment")
{
FileName = string.Format("Dump of {0}, {1}.ravendump", this.DatabaseName, DateTime.Now.ToString("yyyy-MM-dd HH-mm", CultureInfo.InvariantCulture))
};
return new CompletedTask<HttpResponseMessage>(result);
}
public async Task CanDumpAttachmentsEmpty_Dumper()
{
var backupPath = NewDataPath("BackupFolder");
using (var store = NewDocumentStore())
{
var dumper = new DataDumper(store.SystemDatabase) {SmugglerOptions = {Incremental = true, BatchSize = 100, Limit = 206}};
await dumper.ExportData(new SmugglerExportOptions { ToFile = backupPath } );
}
VerifyDump(backupPath, store =>
{
Assert.Equal(0, store.DatabaseCommands.GetAttachmentHeadersStartingWith("user", 0, 500).Count());
});
IOExtensions.DeleteDirectory(backupPath);
}
public async Task CanDumpEmptyDatabase_Dumper()
{
var backupPath = NewDataPath("BackupFolder");
using (var server = GetNewServer(databaseName: Constants.SystemDatabase))
{
using (new DocumentStore { Url = "http://localhost:8079" }.Initialize())
{
// now perform full backup
var dumper = new DataDumper(server.SystemDatabase) {SmugglerOptions = {Incremental = true}};
await dumper.ExportData(new SmugglerExportOptions { ToFile = backupPath });
}
}
VerifyDump(backupPath, store => Assert.Equal(0, store.SystemDatabase.Documents.GetDocuments(0, int.MaxValue, null, CancellationToken.None).Count()));
IOExtensions.DeleteDirectory(backupPath);
}
public async Task CanPerformDumpWithLimit_Dumper()
{
var backupPath = NewDataPath("BackupFolder");
using (var store = NewDocumentStore())
{
InsertUsers(store, 0, 2000);
var dumper = new DataDumper(store.SystemDatabase) {SmugglerOptions = {Limit = 1500, Incremental = true}};
dumper.SmugglerOptions.Filters.Add(
new FilterSetting
{
Path = "@metadata.Raven-Entity-Name",
Values = {"Users"},
ShouldMatch = true,
});
await dumper.ExportData(new SmugglerExportOptions {ToFile = backupPath});
}
VerifyDump(backupPath, store =>
{
using (var session = store.OpenSession())
{
Assert.Equal(1500, session.Query<User>().Customize(x => x.WaitForNonStaleResultsAsOfNow()).Count());
}
});
IOExtensions.DeleteDirectory(backupPath);
}
public async Task CanDumpAttachments_Dumper()
{
var backupPath = NewDataPath("BackupFolder");
using (var store = NewDocumentStore())
{
InsertAttachments(store, 328);
var dumper = new DataDumper(store.DocumentDatabase);
await dumper.ExportData(new SmugglerExportOptions { ToFile = backupPath }, new SmugglerOptions { Incremental = true, BatchSize = 100 });
}
VerifyDump(backupPath, store => Assert.Equal(328, store.DatabaseCommands.GetAttachmentHeadersStartingWith("user", 0, 500).Count()));
IOExtensions.DeleteDirectory(backupPath);
}