public void CanBackupToDirectory_MultipleBackups()
{
var backupPath = NewDataPath("BackupFolder");
using (var store = NewDocumentStore())
{
using (var session = store.OpenSession())
{
session.Store(new User {
Name = "oren"
});
var periodicBackupSetup = new PeriodicBackupSetup
{
LocalFolderName = backupPath,
IntervalMilliseconds = 25
};
session.Store(periodicBackupSetup, PeriodicBackupSetup.RavenDocumentKey);
session.SaveChanges();
}
SpinWait.SpinUntil(() =>
{
var jsonDocument = store.DatabaseCommands.Get(PeriodicBackupStatus.RavenDocumentKey);
if (jsonDocument == null)
{
return(false);
}
var periodicBackupStatus = jsonDocument.DataAsJson.JsonDeserialization <PeriodicBackupStatus>();
return(periodicBackupStatus.LastDocsEtag != Etag.Empty && periodicBackupStatus.LastDocsEtag != null);
});
var etagForBackups = store.DatabaseCommands.Get(PeriodicBackupStatus.RavenDocumentKey).Etag;
using (var session = store.OpenSession())
{
session.Store(new User {
Name = "ayende"
});
session.SaveChanges();
}
SpinWait.SpinUntil(() =>
store.DatabaseCommands.Get(PeriodicBackupStatus.RavenDocumentKey).Etag != etagForBackups);
}
using (var store = NewDocumentStore())
{
var smugglerOptions = new SmugglerOptions
{
BackupPath = backupPath
};
var dataDumper = new DataDumper(store.DocumentDatabase, smugglerOptions);
dataDumper.ImportData(smugglerOptions, true).Wait();
using (var session = store.OpenSession())
{
Assert.Equal("oren", session.Load <User>(1).Name);
Assert.Equal("ayende", session.Load <User>(2).Name);
}
}
IOExtensions.DeleteDirectory(backupPath);
}
public override void CheckForChanges()
{
if (HasUnsavedChanges)
{
return;
}
if (PeriodicBackupSettings.Equals(OriginalPeriodicBackupSettings) == false)
{
HasUnsavedChanges = true;
return;
}
if (PeriodicBackupSetup == null)
{
if (OriginalPeriodicBackupSetup == null)
{
return;
}
HasUnsavedChanges = true;
return;
}
if (PeriodicBackupSetup.Equals(OriginalPeriodicBackupSetup) == false)
{
HasUnsavedChanges = true;
}
}
private void UploadToGlacier(string backupPath, PeriodicBackupSetup localBackupConfigs)
{
var awsRegion = RegionEndpoint.GetBySystemName(localBackupConfigs.AwsRegionEndpoint) ?? RegionEndpoint.USEast1;
var manager = new ArchiveTransferManager(awsAccessKey, awsSecretKey, awsRegion);
var archiveId = manager.Upload(localBackupConfigs.GlacierVaultName, GetArchiveDescription(), backupPath).ArchiveId;
logger.Info(string.Format("Successfully uploaded backup {0} to Glacier, archive ID: {1}", Path.GetFileName(backupPath),
archiveId));
}
public void CanBackupToDirectory_MultipleBackups()
{
var backupPath = GetPath("BackupFolder");
using (var store = NewDocumentStore())
{
Guid?etagForBackups;
using (var session = store.OpenSession())
{
session.Store(new User {
Name = "oren"
});
var periodicBackupSetup = new PeriodicBackupSetup
{
LocalFolderName = backupPath,
IntervalMilliseconds = 25
};
session.Store(periodicBackupSetup, PeriodicBackupSetup.RavenDocumentKey);
session.SaveChanges();
etagForBackups = session.Advanced.GetEtagFor(periodicBackupSetup);
}
SpinWait.SpinUntil(() =>
store.DatabaseCommands.Get(PeriodicBackupSetup.RavenDocumentKey).Etag != etagForBackups);
etagForBackups = store.DatabaseCommands.Get(PeriodicBackupSetup.RavenDocumentKey).Etag;
using (var session = store.OpenSession())
{
session.Store(new User {
Name = "ayende"
});
session.SaveChanges();
}
SpinWait.SpinUntil(() =>
store.DatabaseCommands.Get(PeriodicBackupSetup.RavenDocumentKey).Etag != etagForBackups);
}
using (var store = NewDocumentStore())
{
var smugglerOptions = new SmugglerOptions
{
BackupPath = backupPath
};
var dataDumper = new DataDumper(store.DocumentDatabase, smugglerOptions);
dataDumper.ImportData(smugglerOptions, true);
using (var session = store.OpenSession())
{
Assert.Equal("oren", session.Load <User>(1).Name);
Assert.Equal("ayende", session.Load <User>(2).Name);
}
}
IOExtensions.DeleteDirectory(backupPath);
}
private void UploadToServer(string backupPath, PeriodicBackupSetup localBackupConfigs)
{
if (!string.IsNullOrWhiteSpace(localBackupConfigs.GlacierVaultName))
{
UploadToGlacier(backupPath, localBackupConfigs);
}
else if (!string.IsNullOrWhiteSpace(localBackupConfigs.S3BucketName))
{
UploadToS3(backupPath, localBackupConfigs);
}
}
private void ReadSetupValuesFromDocument()
{
using (LogContext.WithDatabase(Database.Name))
{
try
{
// Not having a setup doc means this DB isn't enabled for periodic backups
var document = Database.Get(PeriodicBackupSetup.RavenDocumentKey, null);
if (document == null)
{
backupConfigs = null;
backupStatus = null;
return;
}
var status = Database.Get(PeriodicBackupStatus.RavenDocumentKey, null);
backupStatus = status == null ? new PeriodicBackupStatus() : status.DataAsJson.JsonDeserialization <PeriodicBackupStatus>();
backupConfigs = document.DataAsJson.JsonDeserialization <PeriodicBackupSetup>();
if (backupConfigs.IntervalMilliseconds <= 0)
{
logger.Warn("Periodic backup interval is set to zero or less, periodic backup is now disabled");
return;
}
awsAccessKey = Database.Configuration.Settings["Raven/AWSAccessKey"];
awsSecretKey = Database.Configuration.Settings["Raven/AWSSecretKey"];
azureStorageAccount = Database.Configuration.Settings["Raven/AzureStorageAccount"];
azureStorageKey = Database.Configuration.Settings["Raven/AzureStorageKey"];
var interval = TimeSpan.FromMilliseconds(backupConfigs.IntervalMilliseconds);
logger.Info("Periodic backups started, will backup every" + interval.TotalMinutes + "minutes");
var timeSinceLastBackup = DateTime.UtcNow - backupStatus.LastBackup;
var nextBackup = timeSinceLastBackup >= interval ? TimeSpan.Zero : interval - timeSinceLastBackup;
timer = new Timer(TimerCallback, null, nextBackup, interval);
}
catch (Exception ex)
{
logger.ErrorException("Could not read periodic backup config", ex);
Database.AddAlert(new Alert
{
AlertLevel = AlertLevel.Error,
CreatedAt = SystemTime.UtcNow,
Message = ex.Message,
Title = "Could not read periodic backup config",
Exception = ex.ToString(),
UniqueKey = "Periodic Backup Config Error"
});
}
}
}
private void ReadSetupValuesFromDocument()
{
using (LogManager.OpenMappedContext("database", Database.Name ?? Constants.SystemDatabase))
using (new DisposableAction(() => LogContext.DatabaseName.Value = null))
{
LogContext.DatabaseName.Value = Database.Name;
try
{
// Not having a setup doc means this DB isn't enabled for periodic backups
var document = Database.Get(PeriodicBackupSetup.RavenDocumentKey, null);
if (document == null)
{
backupConfigs = null;
return;
}
backupConfigs = document.DataAsJson.JsonDeserialization <PeriodicBackupSetup>();
if (backupConfigs.IntervalMilliseconds <= 0)
{
logger.Warn("Peridoic backup interval is set to zero or less, periodic backup is now disabled");
return;
}
awsAccessKey = Database.Configuration.Settings["Raven/AWSAccessKey"];
awsSecretKey = Database.Configuration.Settings["Raven/AWSSecretKey"];
var interval = TimeSpan.FromMilliseconds(backupConfigs.IntervalMilliseconds);
logger.Info("Periodic backups started, will backup every" + interval.TotalMinutes + "minutes");
timer = new Timer(TimerCallback, null, TimeSpan.Zero, interval);
}
catch (Exception ex)
{
logger.ErrorException("Could not read periodic backup config", ex);
Database.AddAlert(new Alert
{
AlertLevel = AlertLevel.Error,
CreatedAt = SystemTime.UtcNow,
Message = ex.Message,
Title = "Error in Periodic Backup",
Exception = ex
});
}
}
}
private void UploadToS3(string backupPath, PeriodicBackupSetup localBackupConfigs)
{
var awsRegion = RegionEndpoint.GetBySystemName(localBackupConfigs.AwsRegionEndpoint) ?? RegionEndpoint.USEast1;
using (var client = new Amazon.S3.AmazonS3Client(awsAccessKey, awsSecretKey, awsRegion))
using (var fileStream = File.OpenRead(backupPath))
{
var key = Path.GetFileName(backupPath);
var request = new PutObjectRequest();
request.WithMetaData("Description", GetArchiveDescription());
request.WithInputStream(fileStream);
request.WithBucketName(localBackupConfigs.S3BucketName);
request.WithKey(key);
using (client.PutObject(request))
{
logger.Info(string.Format("Successfully uploaded backup {0} to S3 bucket {1}, with key {2}",
Path.GetFileName(backupPath), localBackupConfigs.S3BucketName, key));
}
}
}
public void CanBackupToDirectory()
{
var backupPath = NewDataPath("BackupFolder");
using (var store = NewDocumentStore())
{
using (var session = store.OpenSession())
{
session.Store(new User {
Name = "oren"
});
var periodicBackupSetup = new PeriodicBackupSetup
{
LocalFolderName = backupPath,
IntervalMilliseconds = 25
};
session.Store(periodicBackupSetup, PeriodicBackupSetup.RavenDocumentKey);
session.SaveChanges();
}
SpinWait.SpinUntil(() =>
store.DatabaseCommands.Get(PeriodicBackupStatus.RavenDocumentKey) != null, 10000);
}
using (var store = NewDocumentStore())
{
var smugglerOptions = new SmugglerOptions
{
BackupPath = backupPath
};
var dataDumper = new DataDumper(store.DocumentDatabase, smugglerOptions);
dataDumper.ImportData(smugglerOptions, true).Wait();
using (var session = store.OpenSession())
{
Assert.Equal("oren", session.Load <User>(1).Name);
}
}
IOExtensions.DeleteDirectory(backupPath);
}
public static void ConfigurePeriodicBackup(this IDocumentStore store, ErrorditeConfiguration configuration, string databaseName)
{
using (var session = store.OpenSession(databaseName))
{
session.Advanced.UseOptimisticConcurrency = true;
var backupDoc = session.Load <PeriodicBackupSetup>(PeriodicBackupSetup.RavenDocumentKey);
if (backupDoc == null)
{
backupDoc = new PeriodicBackupSetup
{
IntervalMilliseconds = configuration.RavenBackupInterval,
AwsRegionEndpoint = RegionEndpoint.EUWest1.SystemName,
S3BucketName = databaseName,
};
session.Store(backupDoc, PeriodicBackupSetup.RavenDocumentKey);
session.SaveChanges();
}
}
}
private void DoUpload(string backupPath, PeriodicBackupSetup backupConfigs)
{
var AWSRegion = RegionEndpoint.GetBySystemName(backupConfigs.AwsRegionEndpoint) ?? RegionEndpoint.USEast1;
var desc = string.Format("Raven.Database.Backup {0} {1}", Database.Name,
DateTimeOffset.UtcNow.ToString("u"));
if (!string.IsNullOrWhiteSpace(backupConfigs.GlacierVaultName))
{
var manager = new ArchiveTransferManager(awsAccessKey, awsSecretKey, AWSRegion);
var archiveId = manager.Upload(backupConfigs.GlacierVaultName, desc, backupPath).ArchiveId;
logger.Info(string.Format("Successfully uploaded backup {0} to Glacier, archive ID: {1}", Path.GetFileName(backupPath),
archiveId));
return;
}
if (!string.IsNullOrWhiteSpace(backupConfigs.S3BucketName))
{
var client = new Amazon.S3.AmazonS3Client(awsAccessKey, awsSecretKey, AWSRegion);
using (var fileStream = File.OpenRead(backupPath))
{
var key = Path.GetFileName(backupPath);
var request = new PutObjectRequest();
request.WithMetaData("Description", desc);
request.WithInputStream(fileStream);
request.WithBucketName(backupConfigs.S3BucketName);
request.WithKey(key);
using (S3Response _ = client.PutObject(request))
{
logger.Info(string.Format("Successfully uploaded backup {0} to S3 bucket {1}, with key {2}",
Path.GetFileName(backupPath), backupConfigs.S3BucketName, key));
return;
}
}
}
}
private void UploadToAzure(string backupPath, PeriodicBackupSetup localBackupConfigs)
{
StorageCredentials storageCredentials = new StorageCredentials(azureStorageAccount, azureStorageKey);
CloudStorageAccount storageAccount = new CloudStorageAccount(storageCredentials, true);
CloudBlobClient blobClient = new CloudBlobClient(storageAccount.BlobEndpoint, storageCredentials);
CloudBlobContainer backupContainer = blobClient.GetContainerReference(localBackupConfigs.AzureStorageContainer);
backupContainer.CreateIfNotExists();
using (var fileStream = File.OpenRead(backupPath))
{
var key = Path.GetFileName(backupPath);
CloudBlockBlob backupBlob = backupContainer.GetBlockBlobReference(key);
backupBlob.Metadata.Add("Description", this.GetArchiveDescription());
backupBlob.UploadFromStream(fileStream);
backupBlob.SetMetadata();
this.logger.Info(string.Format(
"Successfully uploaded backup {0} to Azure container {1}, with key {2}",
Path.GetFileName(backupPath),
localBackupConfigs.AzureStorageContainer,
key));
}
}
public override void LoadFor(DatabaseDocument document)
{
PeriodicBackupSettings = new PeriodicBackupSettings();
var session = ApplicationModel.Current.Server.Value.DocumentStore
.OpenAsyncSession(ApplicationModel.Current.Server.Value.SelectedDatabase.Value.Name);
if (document.Settings.ContainsKey("Raven/AWSAccessKey") && document.SecuredSettings.ContainsKey("Raven/AWSSecretKey"))
{
PeriodicBackupSettings.AwsAccessKey = document.Settings["Raven/AWSAccessKey"];
PeriodicBackupSettings.AwsSecretKey = document.SecuredSettings["Raven/AWSSecretKey"];
}
if (document.Settings.ContainsKey("Raven/AzureStorageAccount") && document.SecuredSettings.ContainsKey("Raven/AzureStorageKey"))
{
PeriodicBackupSettings.AzureStorageAccount = document.Settings["Raven/AzureStorageAccount"];
PeriodicBackupSettings.AzureStorageKey = document.SecuredSettings["Raven/AzureStorageKey"];
}
OriginalPeriodicBackupSettings = new PeriodicBackupSettings
{
AwsAccessKey = PeriodicBackupSettings.AwsAccessKey,
AwsSecretKey = PeriodicBackupSettings.AwsSecretKey,
AzureStorageAccount = PeriodicBackupSettings.AzureStorageAccount,
AzureStorageKey = PeriodicBackupSettings.AzureStorageKey
};
session.LoadAsync <PeriodicBackupSetup>(PeriodicBackupSetup.RavenDocumentKey).ContinueWith(task =>
{
PeriodicBackupSetup = task.Result;
if (PeriodicBackupSetup == null)
{
return;
}
OriginalPeriodicBackupSetup = new PeriodicBackupSetup
{
AwsRegionEndpoint = PeriodicBackupSetup.AwsRegionEndpoint,
AzureStorageContainer = PeriodicBackupSetup.AzureStorageContainer,
GlacierVaultName = PeriodicBackupSetup.GlacierVaultName,
IntervalMilliseconds = PeriodicBackupSetup.IntervalMilliseconds,
LocalFolderName = PeriodicBackupSetup.LocalFolderName,
S3BucketName = PeriodicBackupSetup.S3BucketName
};
HasDocument = true;
if (string.IsNullOrWhiteSpace(PeriodicBackupSetup.LocalFolderName) == false)
{
SelectedOption.Value = 0;
}
else if (string.IsNullOrWhiteSpace(PeriodicBackupSetup.GlacierVaultName) == false)
{
SelectedOption.Value = 1;
}
else if (string.IsNullOrWhiteSpace(PeriodicBackupSetup.S3BucketName) == false)
{
SelectedOption.Value = 2;
}
else if (string.IsNullOrWhiteSpace(PeriodicBackupSetup.AzureStorageContainer) == false)
{
SelectedOption.Value = 3;
}
OnPropertyChanged(() => HasDocument);
OnPropertyChanged(() => PeriodicBackupSetup);
});
}
