private void ReadSetupValuesFromDocument()
{
using (LogContext.WithDatabase(Database.Name))
{
try
{
// Not having a setup doc means this DB isn't enabled for periodic backups
var document = Database.Get(PeriodicBackupSetup.RavenDocumentKey, null);
if (document == null)
{
backupConfigs = null;
backupStatus = null;
return;
}
var status = Database.Get(PeriodicBackupStatus.RavenDocumentKey, null);
backupStatus = status == null ? new PeriodicBackupStatus() : status.DataAsJson.JsonDeserialization<PeriodicBackupStatus>();
backupConfigs = document.DataAsJson.JsonDeserialization<PeriodicBackupSetup>();
if (backupConfigs.IntervalMilliseconds <= 0)
{
logger.Warn("Periodic backup interval is set to zero or less, periodic backup is now disabled");
return;
}
awsAccessKey = Database.Configuration.Settings["Raven/AWSAccessKey"];
awsSecretKey = Database.Configuration.Settings["Raven/AWSSecretKey"];
azureStorageAccount = Database.Configuration.Settings["Raven/AzureStorageAccount"];
azureStorageKey = Database.Configuration.Settings["Raven/AzureStorageKey"];
var interval = TimeSpan.FromMilliseconds(backupConfigs.IntervalMilliseconds);
logger.Info("Periodic backups started, will backup every" + interval.TotalMinutes + "minutes");
var timeSinceLastBackup = DateTime.UtcNow - backupStatus.LastBackup;
var nextBackup = timeSinceLastBackup >= interval ? TimeSpan.Zero : interval - timeSinceLastBackup;
timer = new Timer(TimerCallback, null, nextBackup, interval);
}
catch (Exception ex)
{
logger.ErrorException("Could not read periodic backup config", ex);
Database.AddAlert(new Alert
{
AlertLevel = AlertLevel.Error,
CreatedAt = SystemTime.UtcNow,
Message = ex.Message,
Title = "Could not read periodic backup config",
Exception = ex.ToString(),
UniqueKey = "Periodic Backup Config Error"
});
}
}
}
public async Task CanDumpWhenHiddenDocsWithLimit_Smuggler()
{
var backupPath = NewDataPath("BackupFolder");
using (GetNewServer())
{
using (var store = new DocumentStore { Url = "http://localhost:8079" }.Initialize())
{
InsertHidenUsers(store, 2000);
var user1 = store.DatabaseCommands.Get("users/1");
Assert.Null(user1);
InsertUsers(store, 1, 25);
// now perform full backup
var options = new SmugglerOptions
{
BackupPath = backupPath,
};
var dumper = new SmugglerApi(options, new RavenConnectionStringOptions
{
Url = "http://localhost:8079",
});
var backupStatus = new PeriodicBackupStatus();
await dumper.ExportData(null, null, true, backupStatus);
}
}
VerifyDump(backupPath, store =>
{
using (var session = store.OpenSession())
{
Assert.Equal(25, session.Query<User>().Customize(x => x.WaitForNonStaleResultsAsOfNow()).Count());
}
});
IOExtensions.DeleteDirectory(backupPath);
}
public async Task CanDumpEmptyDatabase_Smuggler()
{
var backupPath = NewDataPath("BackupFolder");
using (NewRemoteDocumentStore())
{
// now perform full backup
var options = new SmugglerOptions
{
BackupPath = backupPath,
};
var dumper = new SmugglerApi(options, new RavenConnectionStringOptions
{
Url = "http://localhost:8079",
});
var backupStatus = new PeriodicBackupStatus();
await dumper.ExportData(null, null, true, backupStatus);
}
VerifyDump(backupPath, store => Assert.Equal(0, store.DocumentDatabase.GetDocuments(0, int.MaxValue, null, CancellationToken.None).Count()));
IOExtensions.DeleteDirectory(backupPath);
}
public async Task CanDumpEmptyDatabase_Dumper()
{
var backupPath = NewDataPath("BackupFolder");
using (var server = GetNewServer())
{
using (new DocumentStore { Url = "http://localhost:8079" }.Initialize())
{
// now perform full backup
var options = new SmugglerOptions
{
BackupPath = backupPath,
};
var dumper = new DataDumper(server.Database, options);
var backupStatus = new PeriodicBackupStatus();
await dumper.ExportData(null, null, true, backupStatus);
}
}
VerifyDump(backupPath, store => Assert.Equal(0, store.DocumentDatabase.GetDocuments(0, int.MaxValue, null, CancellationToken.None).Count()));
IOExtensions.DeleteDirectory(backupPath);
}
public async Task CanPerformDumpWithLimitAndFilter_Smuggler()
{
var backupPath = NewDataPath("BackupFolder");
using (var store = NewRemoteDocumentStore())
{
var counter = 0;
counter = InsertUsers(store, counter, 1000);
counter = InsertDevelopers(store, counter, 2);
counter = InsertUsers(store, counter, 1000);
InsertDevelopers(store, counter, 2);
WaitForIndexing(store);
var options = new SmugglerOptions
{
Limit = 5,
BackupPath = backupPath,
Filters =
{
new FilterSetting
{
Path = "@metadata.Raven-Entity-Name",
Values = {"Developers"},
ShouldMatch = true,
}
}
};
var dumper = new SmugglerApi(options, new RavenConnectionStringOptions
{
Url = "http://localhost:8079",
});
var backupStatus = new PeriodicBackupStatus();
await dumper.ExportData(null, null, true, backupStatus);
}
VerifyDump(backupPath, store =>
{
using (var session = store.OpenSession())
{
Assert.Equal(4, session.Query<Developer>().Customize(x => x.WaitForNonStaleResultsAsOfNow()).Count());
}
});
IOExtensions.DeleteDirectory(backupPath);
}
public async Task CanPerformDumpWithLimit_Dumper()
{
var backupPath = NewDataPath("BackupFolder");
using (var store = NewDocumentStore())
{
InsertUsers(store, 0, 2000);
var options = new SmugglerOptions
{
Limit = 1500,
BackupPath = backupPath,
Filters =
{
new FilterSetting
{
Path = "@metadata.Raven-Entity-Name",
Values = {"Users"},
ShouldMatch = true,
}
}
};
var dumper = new DataDumper(store.DocumentDatabase, options);
var backupStatus = new PeriodicBackupStatus();
await dumper.ExportData(null, null, true, backupStatus);
}
VerifyDump(backupPath, store =>
{
using (var session = store.OpenSession())
{
Assert.Equal(1500, session.Query<User>().Customize(x => x.WaitForNonStaleResultsAsOfNow()).Count());
}
});
IOExtensions.DeleteDirectory(backupPath);
}
public async Task CanPerformDump_Smuggler()
{
var backupPath = NewDataPath("BackupFolder");
using (var store = NewRemoteDocumentStore())
{
InsertUsers(store, 0, 2000);
var options = new SmugglerOptions
{
BackupPath = backupPath,
};
var dumper = new SmugglerApi(options, new RavenConnectionStringOptions
{
Url = "http://localhost:8079",
});
var backupStatus = new PeriodicBackupStatus();
await dumper.ExportData(null, null, true, backupStatus);
}
VerifyDump(backupPath, store =>
{
using (var session = store.OpenSession())
{
Assert.Equal(2000, session.Query<User>().Customize(x => x.WaitForNonStaleResultsAsOfNow()).Count());
}
});
IOExtensions.DeleteDirectory(backupPath);
}
protected void WaitForPeriodicBackup(DocumentDatabase db, PeriodicBackupStatus previousStatus)
{
WaitForPeriodicBackup(key => db.Documents.Get(key, null), previousStatus);
}
public override async Task<string> ExportData(Stream stream, SmugglerOptions options, bool incremental, PeriodicBackupStatus backupStatus = null)
{
using (store = CreateStore())
{
return await base.ExportData(stream, options, incremental, backupStatus);
}
}
public virtual Task<string> ExportData(Stream stream, SmugglerOptions options, bool incremental, PeriodicBackupStatus backupStatus = null)
{
return ExportData(stream, options, incremental, true, backupStatus);
}
public virtual async Task<string> ExportData(Stream stream, SmugglerOptions options, bool incremental, bool lastEtagsFromFile, PeriodicBackupStatus backupStatus)
{
options = options ?? SmugglerOptions;
if (options == null)
throw new ArgumentNullException("options");
var file = options.BackupPath;
#if !SILVERLIGHT
if (incremental)
{
if (Directory.Exists(options.BackupPath) == false)
{
if (File.Exists(options.BackupPath))
options.BackupPath = Path.GetDirectoryName(options.BackupPath) ?? options.BackupPath;
else
Directory.CreateDirectory(options.BackupPath);
}
if (lastEtagsFromFile && backupStatus == null) ReadLastEtagsFromFile(options);
if (backupStatus != null) ReadLastEtagsFromClass(options, backupStatus);
file = Path.Combine(options.BackupPath, SystemTime.UtcNow.ToString("yyyy-MM-dd-HH-mm", CultureInfo.InvariantCulture) + ".ravendb-incremental-dump");
if (File.Exists(file))
{
var counter = 1;
while (true)
{
file = Path.Combine(options.BackupPath, SystemTime.UtcNow.ToString("yyyy-MM-dd-HH-mm", CultureInfo.InvariantCulture) + " - " + counter + ".ravendb-incremental-dump");
if (File.Exists(file) == false)
break;
counter++;
}
}
}
#else
if(incremental)
throw new NotSupportedException("Incremental exports are not supported in SL.");
#endif
await DetectServerSupportedFeatures();
SmugglerExportException lastException = null;
bool ownedStream = stream == null;
try
{
stream = stream ?? File.Create(file);
using (var gZipStream = new GZipStream(stream, CompressionMode.Compress,
#if SILVERLIGHT
CompressionLevel.BestCompression,
#endif
leaveOpen: true))
using (var streamWriter = new StreamWriter(gZipStream))
{
var jsonWriter = new JsonTextWriter(streamWriter)
{
Formatting = Formatting.Indented
};
jsonWriter.WriteStartObject();
jsonWriter.WritePropertyName("Indexes");
jsonWriter.WriteStartArray();
if ((options.OperateOnTypes & ItemType.Indexes) == ItemType.Indexes)
{
await ExportIndexes(jsonWriter);
}
jsonWriter.WriteEndArray();
jsonWriter.WritePropertyName("Docs");
jsonWriter.WriteStartArray();
if (options.OperateOnTypes.HasFlag(ItemType.Documents))
{
try
{
options.LastDocsEtag = await ExportDocuments(options, jsonWriter, options.LastDocsEtag);
}
catch (SmugglerExportException e)
{
options.LastDocsEtag = e.LastEtag;
e.File = file;
lastException = e;
}
}
jsonWriter.WriteEndArray();
jsonWriter.WritePropertyName("Attachments");
jsonWriter.WriteStartArray();
if (options.OperateOnTypes.HasFlag(ItemType.Attachments) && lastException == null)
{
try
{
options.LastAttachmentEtag = await ExportAttachments(jsonWriter, options.LastAttachmentEtag);
}
catch (SmugglerExportException e)
{
options.LastAttachmentEtag = e.LastEtag;
e.File = file;
lastException = e;
}
}
jsonWriter.WriteEndArray();
jsonWriter.WritePropertyName("Transformers");
jsonWriter.WriteStartArray();
if (options.OperateOnTypes.HasFlag(ItemType.Transformers) && lastException == null)
{
await ExportTransformers(jsonWriter);
}
jsonWriter.WriteEndArray();
jsonWriter.WriteEndObject();
streamWriter.Flush();
}
#if !SILVERLIGHT
if (incremental && lastEtagsFromFile)
WriteLastEtagsFromFile(options);
#endif
if (lastException != null)
throw lastException;
return file;
}
finally
{
if (ownedStream && stream != null)
stream.Dispose();
}
}
private void ReadLastEtagsFromClass(SmugglerOptions options, PeriodicBackupStatus backupStatus)
{
options.LastAttachmentEtag = backupStatus.LastAttachmentsEtag;
options.LastDocsEtag = backupStatus.LastDocsEtag;
}
private void WaitForPeriodicBackup(Func<string, JsonDocument> getDocument, PeriodicBackupStatus previousStatus)
{
PeriodicBackupStatus currentStatus = null;
var done = SpinWait.SpinUntil(() =>
{
currentStatus = GetPerodicBackupStatus(getDocument);
return currentStatus.LastDocsEtag != previousStatus.LastDocsEtag ||
currentStatus.LastAttachmentsEtag != previousStatus.LastAttachmentsEtag ||
currentStatus.LastDocsDeletionEtag != previousStatus.LastDocsDeletionEtag ||
currentStatus.LastAttachmentDeletionEtag != previousStatus.LastAttachmentDeletionEtag;
}, Debugger.IsAttached ? TimeSpan.FromMinutes(120) : TimeSpan.FromMinutes(15));
Assert.True(done);
previousStatus.LastDocsEtag = currentStatus.LastDocsEtag;
previousStatus.LastAttachmentsEtag = currentStatus.LastAttachmentsEtag;
previousStatus.LastDocsDeletionEtag = currentStatus.LastDocsDeletionEtag;
previousStatus.LastAttachmentDeletionEtag = currentStatus.LastAttachmentDeletionEtag;
}
protected void WaitForPeriodicBackup(IDatabaseCommands commands, PeriodicBackupStatus previousStatus)
{
WaitForPeriodicBackup(commands.Get, previousStatus);
}
public async Task CanDumpAttachmentsEmpty_Dumper()
{
var backupPath = NewDataPath("BackupFolder");
using (var store = NewDocumentStore())
{
var options = new SmugglerOptions
{
BackupPath = backupPath,
BatchSize = 100,
Limit = 206
};
var dumper = new DataDumper(store.DocumentDatabase, options);
var backupStatus = new PeriodicBackupStatus();
await dumper.ExportData(null, null, true, backupStatus);
}
VerifyDump(backupPath, store =>
{
Assert.Equal(0, store.DatabaseCommands.GetAttachmentHeadersStartingWith("user", 0, 500).Count());
});
IOExtensions.DeleteDirectory(backupPath);
}
public override async Task<string> ExportData(Stream stream, SmugglerOptions options, bool incremental, bool lastEtagsFromFile, PeriodicBackupStatus lastEtag)
{
using (store = CreateStore())
{
return await base.ExportData(stream, options, incremental, lastEtagsFromFile, lastEtag);
}
}
public async Task CanDumpAttachmentsEmpty_Smuggler()
{
var backupPath = NewDataPath("BackupFolder");
using (NewRemoteDocumentStore())
{
var options = new SmugglerOptions
{
BackupPath = backupPath,
BatchSize = 100,
Limit = 206
};
var dumper = new SmugglerApi(options, new RavenConnectionStringOptions
{
Url = "http://localhost:8079",
});
var backupStatus = new PeriodicBackupStatus();
await dumper.ExportData(null, null, true, backupStatus);
}
VerifyDump(backupPath, store =>
{
Assert.Equal(0, store.DatabaseCommands.GetAttachmentHeadersStartingWith("user", 0, 500).Count());
});
IOExtensions.DeleteDirectory(backupPath);
}
public void WaitForPeriodicBackup(DocumentDatabase db, PeriodicBackupStatus previousStatus, Func<PeriodicBackupStatus, Etag> compareSelector)
{
PeriodicBackupStatus currentStatus = null;
var done = SpinWait.SpinUntil(() =>
{
currentStatus = GetPerodicBackupStatus(db);
return compareSelector(currentStatus) != compareSelector(previousStatus);
}, Debugger.IsAttached ? TimeSpan.FromMinutes(120) : TimeSpan.FromMinutes(15));
Assert.True(done);
previousStatus.LastDocsEtag = currentStatus.LastDocsEtag;
previousStatus.LastAttachmentsEtag = currentStatus.LastAttachmentsEtag;
previousStatus.LastDocsDeletionEtag = currentStatus.LastDocsDeletionEtag;
previousStatus.LastAttachmentDeletionEtag = currentStatus.LastAttachmentDeletionEtag;
}