private void ReadSetupValuesFromDocument() { using (LogContext.WithDatabase(Database.Name)) { try { // Not having a setup doc means this DB isn't enabled for periodic backups var document = Database.Get(PeriodicBackupSetup.RavenDocumentKey, null); if (document == null) { backupConfigs = null; backupStatus = null; return; } var status = Database.Get(PeriodicBackupStatus.RavenDocumentKey, null); backupStatus = status == null ? new PeriodicBackupStatus() : status.DataAsJson.JsonDeserialization<PeriodicBackupStatus>(); backupConfigs = document.DataAsJson.JsonDeserialization<PeriodicBackupSetup>(); if (backupConfigs.IntervalMilliseconds <= 0) { logger.Warn("Periodic backup interval is set to zero or less, periodic backup is now disabled"); return; } awsAccessKey = Database.Configuration.Settings["Raven/AWSAccessKey"]; awsSecretKey = Database.Configuration.Settings["Raven/AWSSecretKey"]; azureStorageAccount = Database.Configuration.Settings["Raven/AzureStorageAccount"]; azureStorageKey = Database.Configuration.Settings["Raven/AzureStorageKey"]; var interval = TimeSpan.FromMilliseconds(backupConfigs.IntervalMilliseconds); logger.Info("Periodic backups started, will backup every" + interval.TotalMinutes + "minutes"); var timeSinceLastBackup = DateTime.UtcNow - backupStatus.LastBackup; var nextBackup = timeSinceLastBackup >= interval ? TimeSpan.Zero : interval - timeSinceLastBackup; timer = new Timer(TimerCallback, null, nextBackup, interval); } catch (Exception ex) { logger.ErrorException("Could not read periodic backup config", ex); Database.AddAlert(new Alert { AlertLevel = AlertLevel.Error, CreatedAt = SystemTime.UtcNow, Message = ex.Message, Title = "Could not read periodic backup config", Exception = ex.ToString(), UniqueKey = "Periodic Backup Config Error" }); } } }
public async Task CanDumpWhenHiddenDocsWithLimit_Smuggler() { var backupPath = NewDataPath("BackupFolder"); using (GetNewServer()) { using (var store = new DocumentStore { Url = "http://localhost:8079" }.Initialize()) { InsertHidenUsers(store, 2000); var user1 = store.DatabaseCommands.Get("users/1"); Assert.Null(user1); InsertUsers(store, 1, 25); // now perform full backup var options = new SmugglerOptions { BackupPath = backupPath, }; var dumper = new SmugglerApi(options, new RavenConnectionStringOptions { Url = "http://localhost:8079", }); var backupStatus = new PeriodicBackupStatus(); await dumper.ExportData(null, null, true, backupStatus); } } VerifyDump(backupPath, store => { using (var session = store.OpenSession()) { Assert.Equal(25, session.Query<User>().Customize(x => x.WaitForNonStaleResultsAsOfNow()).Count()); } }); IOExtensions.DeleteDirectory(backupPath); }
public async Task CanDumpEmptyDatabase_Smuggler() { var backupPath = NewDataPath("BackupFolder"); using (NewRemoteDocumentStore()) { // now perform full backup var options = new SmugglerOptions { BackupPath = backupPath, }; var dumper = new SmugglerApi(options, new RavenConnectionStringOptions { Url = "http://localhost:8079", }); var backupStatus = new PeriodicBackupStatus(); await dumper.ExportData(null, null, true, backupStatus); } VerifyDump(backupPath, store => Assert.Equal(0, store.DocumentDatabase.GetDocuments(0, int.MaxValue, null, CancellationToken.None).Count())); IOExtensions.DeleteDirectory(backupPath); }
public async Task CanDumpEmptyDatabase_Dumper() { var backupPath = NewDataPath("BackupFolder"); using (var server = GetNewServer()) { using (new DocumentStore { Url = "http://localhost:8079" }.Initialize()) { // now perform full backup var options = new SmugglerOptions { BackupPath = backupPath, }; var dumper = new DataDumper(server.Database, options); var backupStatus = new PeriodicBackupStatus(); await dumper.ExportData(null, null, true, backupStatus); } } VerifyDump(backupPath, store => Assert.Equal(0, store.DocumentDatabase.GetDocuments(0, int.MaxValue, null, CancellationToken.None).Count())); IOExtensions.DeleteDirectory(backupPath); }
public async Task CanPerformDumpWithLimitAndFilter_Smuggler() { var backupPath = NewDataPath("BackupFolder"); using (var store = NewRemoteDocumentStore()) { var counter = 0; counter = InsertUsers(store, counter, 1000); counter = InsertDevelopers(store, counter, 2); counter = InsertUsers(store, counter, 1000); InsertDevelopers(store, counter, 2); WaitForIndexing(store); var options = new SmugglerOptions { Limit = 5, BackupPath = backupPath, Filters = { new FilterSetting { Path = "@metadata.Raven-Entity-Name", Values = {"Developers"}, ShouldMatch = true, } } }; var dumper = new SmugglerApi(options, new RavenConnectionStringOptions { Url = "http://localhost:8079", }); var backupStatus = new PeriodicBackupStatus(); await dumper.ExportData(null, null, true, backupStatus); } VerifyDump(backupPath, store => { using (var session = store.OpenSession()) { Assert.Equal(4, session.Query<Developer>().Customize(x => x.WaitForNonStaleResultsAsOfNow()).Count()); } }); IOExtensions.DeleteDirectory(backupPath); }
public async Task CanPerformDumpWithLimit_Dumper() { var backupPath = NewDataPath("BackupFolder"); using (var store = NewDocumentStore()) { InsertUsers(store, 0, 2000); var options = new SmugglerOptions { Limit = 1500, BackupPath = backupPath, Filters = { new FilterSetting { Path = "@metadata.Raven-Entity-Name", Values = {"Users"}, ShouldMatch = true, } } }; var dumper = new DataDumper(store.DocumentDatabase, options); var backupStatus = new PeriodicBackupStatus(); await dumper.ExportData(null, null, true, backupStatus); } VerifyDump(backupPath, store => { using (var session = store.OpenSession()) { Assert.Equal(1500, session.Query<User>().Customize(x => x.WaitForNonStaleResultsAsOfNow()).Count()); } }); IOExtensions.DeleteDirectory(backupPath); }
public async Task CanPerformDump_Smuggler() { var backupPath = NewDataPath("BackupFolder"); using (var store = NewRemoteDocumentStore()) { InsertUsers(store, 0, 2000); var options = new SmugglerOptions { BackupPath = backupPath, }; var dumper = new SmugglerApi(options, new RavenConnectionStringOptions { Url = "http://localhost:8079", }); var backupStatus = new PeriodicBackupStatus(); await dumper.ExportData(null, null, true, backupStatus); } VerifyDump(backupPath, store => { using (var session = store.OpenSession()) { Assert.Equal(2000, session.Query<User>().Customize(x => x.WaitForNonStaleResultsAsOfNow()).Count()); } }); IOExtensions.DeleteDirectory(backupPath); }
protected void WaitForPeriodicBackup(DocumentDatabase db, PeriodicBackupStatus previousStatus) { WaitForPeriodicBackup(key => db.Documents.Get(key, null), previousStatus); }
public override async Task<string> ExportData(Stream stream, SmugglerOptions options, bool incremental, PeriodicBackupStatus backupStatus = null) { using (store = CreateStore()) { return await base.ExportData(stream, options, incremental, backupStatus); } }
public virtual Task<string> ExportData(Stream stream, SmugglerOptions options, bool incremental, PeriodicBackupStatus backupStatus = null) { return ExportData(stream, options, incremental, true, backupStatus); }
public virtual async Task<string> ExportData(Stream stream, SmugglerOptions options, bool incremental, bool lastEtagsFromFile, PeriodicBackupStatus backupStatus) { options = options ?? SmugglerOptions; if (options == null) throw new ArgumentNullException("options"); var file = options.BackupPath; #if !SILVERLIGHT if (incremental) { if (Directory.Exists(options.BackupPath) == false) { if (File.Exists(options.BackupPath)) options.BackupPath = Path.GetDirectoryName(options.BackupPath) ?? options.BackupPath; else Directory.CreateDirectory(options.BackupPath); } if (lastEtagsFromFile && backupStatus == null) ReadLastEtagsFromFile(options); if (backupStatus != null) ReadLastEtagsFromClass(options, backupStatus); file = Path.Combine(options.BackupPath, SystemTime.UtcNow.ToString("yyyy-MM-dd-HH-mm", CultureInfo.InvariantCulture) + ".ravendb-incremental-dump"); if (File.Exists(file)) { var counter = 1; while (true) { file = Path.Combine(options.BackupPath, SystemTime.UtcNow.ToString("yyyy-MM-dd-HH-mm", CultureInfo.InvariantCulture) + " - " + counter + ".ravendb-incremental-dump"); if (File.Exists(file) == false) break; counter++; } } } #else if(incremental) throw new NotSupportedException("Incremental exports are not supported in SL."); #endif await DetectServerSupportedFeatures(); SmugglerExportException lastException = null; bool ownedStream = stream == null; try { stream = stream ?? File.Create(file); using (var gZipStream = new GZipStream(stream, CompressionMode.Compress, #if SILVERLIGHT CompressionLevel.BestCompression, #endif leaveOpen: true)) using (var streamWriter = new StreamWriter(gZipStream)) { var jsonWriter = new JsonTextWriter(streamWriter) { Formatting = Formatting.Indented }; jsonWriter.WriteStartObject(); jsonWriter.WritePropertyName("Indexes"); jsonWriter.WriteStartArray(); if ((options.OperateOnTypes & ItemType.Indexes) == ItemType.Indexes) { await ExportIndexes(jsonWriter); } jsonWriter.WriteEndArray(); jsonWriter.WritePropertyName("Docs"); jsonWriter.WriteStartArray(); if (options.OperateOnTypes.HasFlag(ItemType.Documents)) { try { options.LastDocsEtag = await ExportDocuments(options, jsonWriter, options.LastDocsEtag); } catch (SmugglerExportException e) { options.LastDocsEtag = e.LastEtag; e.File = file; lastException = e; } } jsonWriter.WriteEndArray(); jsonWriter.WritePropertyName("Attachments"); jsonWriter.WriteStartArray(); if (options.OperateOnTypes.HasFlag(ItemType.Attachments) && lastException == null) { try { options.LastAttachmentEtag = await ExportAttachments(jsonWriter, options.LastAttachmentEtag); } catch (SmugglerExportException e) { options.LastAttachmentEtag = e.LastEtag; e.File = file; lastException = e; } } jsonWriter.WriteEndArray(); jsonWriter.WritePropertyName("Transformers"); jsonWriter.WriteStartArray(); if (options.OperateOnTypes.HasFlag(ItemType.Transformers) && lastException == null) { await ExportTransformers(jsonWriter); } jsonWriter.WriteEndArray(); jsonWriter.WriteEndObject(); streamWriter.Flush(); } #if !SILVERLIGHT if (incremental && lastEtagsFromFile) WriteLastEtagsFromFile(options); #endif if (lastException != null) throw lastException; return file; } finally { if (ownedStream && stream != null) stream.Dispose(); } }
private void ReadLastEtagsFromClass(SmugglerOptions options, PeriodicBackupStatus backupStatus) { options.LastAttachmentEtag = backupStatus.LastAttachmentsEtag; options.LastDocsEtag = backupStatus.LastDocsEtag; }
private void WaitForPeriodicBackup(Func<string, JsonDocument> getDocument, PeriodicBackupStatus previousStatus) { PeriodicBackupStatus currentStatus = null; var done = SpinWait.SpinUntil(() => { currentStatus = GetPerodicBackupStatus(getDocument); return currentStatus.LastDocsEtag != previousStatus.LastDocsEtag || currentStatus.LastAttachmentsEtag != previousStatus.LastAttachmentsEtag || currentStatus.LastDocsDeletionEtag != previousStatus.LastDocsDeletionEtag || currentStatus.LastAttachmentDeletionEtag != previousStatus.LastAttachmentDeletionEtag; }, Debugger.IsAttached ? TimeSpan.FromMinutes(120) : TimeSpan.FromMinutes(15)); Assert.True(done); previousStatus.LastDocsEtag = currentStatus.LastDocsEtag; previousStatus.LastAttachmentsEtag = currentStatus.LastAttachmentsEtag; previousStatus.LastDocsDeletionEtag = currentStatus.LastDocsDeletionEtag; previousStatus.LastAttachmentDeletionEtag = currentStatus.LastAttachmentDeletionEtag; }
protected void WaitForPeriodicBackup(IDatabaseCommands commands, PeriodicBackupStatus previousStatus) { WaitForPeriodicBackup(commands.Get, previousStatus); }
public async Task CanDumpAttachmentsEmpty_Dumper() { var backupPath = NewDataPath("BackupFolder"); using (var store = NewDocumentStore()) { var options = new SmugglerOptions { BackupPath = backupPath, BatchSize = 100, Limit = 206 }; var dumper = new DataDumper(store.DocumentDatabase, options); var backupStatus = new PeriodicBackupStatus(); await dumper.ExportData(null, null, true, backupStatus); } VerifyDump(backupPath, store => { Assert.Equal(0, store.DatabaseCommands.GetAttachmentHeadersStartingWith("user", 0, 500).Count()); }); IOExtensions.DeleteDirectory(backupPath); }
public override async Task<string> ExportData(Stream stream, SmugglerOptions options, bool incremental, bool lastEtagsFromFile, PeriodicBackupStatus lastEtag) { using (store = CreateStore()) { return await base.ExportData(stream, options, incremental, lastEtagsFromFile, lastEtag); } }
public async Task CanDumpAttachmentsEmpty_Smuggler() { var backupPath = NewDataPath("BackupFolder"); using (NewRemoteDocumentStore()) { var options = new SmugglerOptions { BackupPath = backupPath, BatchSize = 100, Limit = 206 }; var dumper = new SmugglerApi(options, new RavenConnectionStringOptions { Url = "http://localhost:8079", }); var backupStatus = new PeriodicBackupStatus(); await dumper.ExportData(null, null, true, backupStatus); } VerifyDump(backupPath, store => { Assert.Equal(0, store.DatabaseCommands.GetAttachmentHeadersStartingWith("user", 0, 500).Count()); }); IOExtensions.DeleteDirectory(backupPath); }
public void WaitForPeriodicBackup(DocumentDatabase db, PeriodicBackupStatus previousStatus, Func<PeriodicBackupStatus, Etag> compareSelector) { PeriodicBackupStatus currentStatus = null; var done = SpinWait.SpinUntil(() => { currentStatus = GetPerodicBackupStatus(db); return compareSelector(currentStatus) != compareSelector(previousStatus); }, Debugger.IsAttached ? TimeSpan.FromMinutes(120) : TimeSpan.FromMinutes(15)); Assert.True(done); previousStatus.LastDocsEtag = currentStatus.LastDocsEtag; previousStatus.LastAttachmentsEtag = currentStatus.LastAttachmentsEtag; previousStatus.LastDocsDeletionEtag = currentStatus.LastDocsDeletionEtag; previousStatus.LastAttachmentDeletionEtag = currentStatus.LastAttachmentDeletionEtag; }