public void CanProperlyReadLastEtagUsingPreviousFormat() { var backupPath = NewDataPath("BackupFolder", forceCreateDir: true); var etagFileLocation = Path.Combine(Path.GetDirectoryName(backupPath), "IncrementalExport.state.json"); using (var streamWriter = new StreamWriter(File.Create(etagFileLocation))) { new RavenJObject { { "LastDocEtag", Etag.Parse("00000000-0000-0000-0000-000000000001").ToString() }, { "LastAttachmentEtag", Etag.Parse("00000000-0000-0000-0000-000000000002").ToString() } }.WriteTo(new JsonTextWriter(streamWriter)); streamWriter.Flush(); } var result = new ExportDataResult { FilePath = backupPath }; SmugglerApiBase.ReadLastEtagsFromFile(result); Assert.Equal("00000000-0000-0000-0000-000000000001", result.LastDocsEtag.ToString()); Assert.Equal("00000000-0000-0000-0000-000000000002", result.LastAttachmentsEtag.ToString()); Assert.Equal(Etag.Empty, result.LastDocDeleteEtag); Assert.Equal(Etag.Empty, result.LastAttachmentsDeleteEtag); }
public void PurgeTombstones(ExportDataResult result) { database.TransactionalStorage.Batch(accessor => { // since remove all before is inclusive, but we want last etag for function FetchCurrentMaxEtags we modify ranges accessor.Lists.RemoveAllBefore(Constants.RavenPeriodicExportsDocsTombstones, result.LastDocDeleteEtag.IncrementBy(-1)); accessor.Lists.RemoveAllBefore(Constants.RavenPeriodicExportsAttachmentsTombstones, result.LastAttachmentsDeleteEtag.IncrementBy(-1)); }); }
public override async Task ExportDeletions(JsonTextWriter jsonWriter, ExportDataResult result, LastEtagsInfo maxEtagsToFetch) { jsonWriter.WritePropertyName("DocsDeletions"); jsonWriter.WriteStartArray(); result.LastDocDeleteEtag = await Operations.ExportDocumentsDeletion(jsonWriter, result.LastDocDeleteEtag, maxEtagsToFetch.LastDocDeleteEtag.IncrementBy(1)); jsonWriter.WriteEndArray(); jsonWriter.WritePropertyName("AttachmentsDeletions"); jsonWriter.WriteStartArray(); result.LastAttachmentsDeleteEtag = await Operations.ExportAttachmentsDeletion(jsonWriter, result.LastAttachmentsDeleteEtag, maxEtagsToFetch.LastAttachmentsDeleteEtag.IncrementBy(1)); jsonWriter.WriteEndArray(); }
public async Task CanHandleDocumentExceptionsGracefully_Smuggler() { var backupPath = NewDataPath("BackupFolder"); var server = GetNewServer(databaseName: Constants.SystemDatabase); var alreadyReset = false; var forwarder = new ProxyServer(8070, 8079) { VetoTransfer = (totalRead, buffer) => { if (alreadyReset == false && totalRead > 25000) { alreadyReset = true; return true; } return false; } }; try { string databaseName; using (var store = new DocumentStore { Url = "http://localhost:8079" }) { databaseName = store.DefaultDatabase; store.Initialize(); InsertUsers(store, 0, 2000); } var dumper = new SmugglerApi {SmugglerOptions = {Limit = 1900, Incremental = true}}; var allDocs = new List<RavenJObject>(); ExportDataResult exportResult = null; try { exportResult = await dumper.ExportData(new SmugglerExportOptions { ToFile = backupPath, From = new RavenConnectionStringOptions { Url = "http://localhost:8070", DefaultDatabase = databaseName, } }); Assert.False(true, "Previous op should throw."); } catch (SmugglerExportException e) { exportResult = new ExportDataResult { FilePath = e.File }; } using (var fileSteam = new FileStream(exportResult.FilePath, FileMode.Open)) using (var stream = new GZipStream(fileSteam, CompressionMode.Decompress)) { var chunk1 = RavenJToken.TryLoad(stream) as RavenJObject; var doc1 = chunk1["Docs"] as RavenJArray; allDocs.AddRange(doc1.Values<RavenJObject>()); } exportResult = await dumper.ExportData(new SmugglerExportOptions { ToFile = backupPath, From = new RavenConnectionStringOptions { Url = "http://localhost:8070", DefaultDatabase = databaseName, } }); using (var fileStream = new FileStream(exportResult.FilePath, FileMode.Open)) using (var stream = new GZipStream(fileStream, CompressionMode.Decompress)) { var chunk2 = RavenJToken.TryLoad(stream) as RavenJObject; var doc2 = chunk2["Docs"] as RavenJArray; allDocs.AddRange(doc2.Values<RavenJObject>()); } Assert.Equal(2000, allDocs.Count(d => (d.Value<string>("Name") ?? String.Empty).StartsWith("User"))); } finally { forwarder.Dispose(); server.Dispose(); IOExtensions.DeleteDirectory(backupPath); } }
public virtual async Task<ExportDataResult> ExportData(SmugglerExportOptions exportOptions) { Operations.Configure(SmugglerOptions); Operations.Initialize(SmugglerOptions); var result = new ExportDataResult { FilePath = exportOptions.ToFile, LastAttachmentsEtag = SmugglerOptions.StartAttachmentsEtag, LastDocsEtag = SmugglerOptions.StartDocsEtag, LastDocDeleteEtag = SmugglerOptions.StartDocsDeletionEtag, LastAttachmentsDeleteEtag = SmugglerOptions.StartAttachmentsDeletionEtag }; if (SmugglerOptions.Incremental) { if (Directory.Exists(result.FilePath) == false) { if (File.Exists(result.FilePath)) result.FilePath = Path.GetDirectoryName(result.FilePath) ?? result.FilePath; else Directory.CreateDirectory(result.FilePath); } if (SmugglerOptions.StartDocsEtag == Etag.Empty && SmugglerOptions.StartAttachmentsEtag == Etag.Empty) { ReadLastEtagsFromFile(result); } result.FilePath = Path.Combine(result.FilePath, SystemTime.UtcNow.ToString("yyyy-MM-dd-HH-mm", CultureInfo.InvariantCulture) + ".ravendb-incremental-dump"); if (File.Exists(result.FilePath)) { var counter = 1; while (true) { // ReSharper disable once AssignNullToNotNullAttribute result.FilePath = Path.Combine(Path.GetDirectoryName(result.FilePath), SystemTime.UtcNow.ToString("yyyy-MM-dd-HH-mm", CultureInfo.InvariantCulture) + " - " + counter + ".ravendb-incremental-dump"); if (File.Exists(result.FilePath) == false) break; counter++; } } } SmugglerExportException lastException = null; bool ownedStream = exportOptions.ToStream == null; var stream = exportOptions.ToStream ?? File.Create(result.FilePath); try { await DetectServerSupportedFeatures(exportOptions.From); } catch (WebException e) { Operations.ShowProgress("Failed to query server for supported features. Reason : " + e.Message); SetLegacyMode(); //could not detect supported features, then run in legacy mode // lastException = new SmugglerExportException // { // LastEtag = Etag.Empty, // File = ownedStream ? result.FilePath : null // }; } try { using (var gZipStream = new GZipStream(stream, CompressionMode.Compress, leaveOpen: true)) using (var streamWriter = new StreamWriter(gZipStream)) { var jsonWriter = new JsonTextWriter(streamWriter) { Formatting = Formatting.Indented }; jsonWriter.WriteStartObject(); jsonWriter.WritePropertyName("Indexes"); jsonWriter.WriteStartArray(); if (SmugglerOptions.OperateOnTypes.HasFlag(ItemType.Indexes)) { await ExportIndexes(exportOptions.From, jsonWriter); } jsonWriter.WriteEndArray(); // used to synchronize max returned values for put/delete operations var maxEtags = Operations.FetchCurrentMaxEtags(); jsonWriter.WritePropertyName("Docs"); jsonWriter.WriteStartArray(); if (SmugglerOptions.OperateOnTypes.HasFlag(ItemType.Documents)) { try { result.LastDocsEtag = await ExportDocuments(exportOptions.From, jsonWriter, result.LastDocsEtag, maxEtags.LastDocsEtag); } catch (SmugglerExportException e) { result.LastDocsEtag = e.LastEtag; e.File = ownedStream ? result.FilePath : null; lastException = e; } } jsonWriter.WriteEndArray(); jsonWriter.WritePropertyName("Attachments"); jsonWriter.WriteStartArray(); if (SmugglerOptions.OperateOnTypes.HasFlag(ItemType.Attachments) && lastException == null) { try { result.LastAttachmentsEtag = await ExportAttachments(exportOptions.From, jsonWriter, result.LastAttachmentsEtag, maxEtags.LastAttachmentsEtag); } catch (SmugglerExportException e) { result.LastAttachmentsEtag = e.LastEtag; e.File = ownedStream ? result.FilePath : null; lastException = e; } } jsonWriter.WriteEndArray(); jsonWriter.WritePropertyName("Transformers"); jsonWriter.WriteStartArray(); if (SmugglerOptions.OperateOnTypes.HasFlag(ItemType.Transformers) && lastException == null) { await ExportTransformers(exportOptions.From, jsonWriter); } jsonWriter.WriteEndArray(); if (SmugglerOptions.ExportDeletions) { await ExportDeletions(jsonWriter, result, maxEtags); } await ExportIdentities(jsonWriter, SmugglerOptions.OperateOnTypes); jsonWriter.WriteEndObject(); streamWriter.Flush(); } if (SmugglerOptions.Incremental) WriteLastEtagsToFile(result, result.FilePath); if (SmugglerOptions.ExportDeletions) { Operations.PurgeTombstones(result); } if (lastException != null) throw lastException; return result; } finally { if (ownedStream && stream != null) stream.Dispose(); } }
public abstract Task ExportDeletions(JsonTextWriter jsonWriter, ExportDataResult result, LastEtagsInfo maxEtagsToFetch);
protected override void PurgeTombstones(ExportDataResult result) { throw new NotImplementedException(); }
public FileContentResult ExportProfile(string startDate, string endDate) { try { List <ExperienceProfileDetail> export = new List <ExperienceProfileDetail>(); string[] paramiters = { WebVisit.DefaultFacetKey, LocaleInfo.DefaultFacetKey, IpInfo.DefaultFacetKey }; var interactionsobj = new RelatedInteractionsExpandOptions(paramiters); DateTime sdate = DateTime.Now; DateTime edate = DateTime.Now; if (!string.IsNullOrEmpty(startDate) && DateTime.TryParse(startDate, out sdate)) { interactionsobj.StartDateTime = sdate.AddDays(1).ToUniversalTime(); } else { interactionsobj.StartDateTime = DateTime.UtcNow.AddDays(-30); } if (!string.IsNullOrEmpty(endDate) && DateTime.TryParse(endDate, out edate)) { interactionsobj.EndDateTime = edate.AddDays(1).ToUniversalTime(); } else { interactionsobj.EndDateTime = DateTime.UtcNow; } interactionsobj.Limit = int.MaxValue; ExportDataResult exportResult; using (Sitecore.XConnect.Client.XConnectClient client = SitecoreXConnectClientConfiguration.GetClient()) { List <Contact> contactList = new List <Contact>(); bool includeAnonymous = false; var settingvalue = Sitecore.Configuration.Settings.GetSetting("IncludeAnonymous"); bool.TryParse(settingvalue, out includeAnonymous); var Contactsid = client.Contacts.Where(d => d.Interactions.Any(i => i.StartDateTime >= interactionsobj.StartDateTime && i.EndDateTime <= interactionsobj.EndDateTime)).AsAsyncQueryable(); if (!includeAnonymous) { Contactsid = Contactsid.Where(c => c.Identifiers.Any(t => t.IdentifierType == Sitecore.XConnect.ContactIdentifierType.Known)); } contactList = Contactsid.ToList().Result; var references = new List <IEntityReference <Sitecore.XConnect.Contact> >(); references.AddRange(contactList); var contacts = client.Get <Contact>(references, new Sitecore.XConnect.ContactExpandOptions(PersonalInformation.DefaultFacetKey) { Interactions = interactionsobj }.Expand <EmailAddressList>().Expand <AddressList>().Expand <PhoneNumberList>()); exportResult = new ExportDataResult() { Content = GenerateFileContent(contacts), FileName = GenerateFileName(interactionsobj.StartDateTime.Value, interactionsobj.EndDateTime.Value), MediaType = "application/octet-stream" }; } FileContentResult fileresult; if (exportResult != null) { fileresult = new FileContentResult(exportResult.Content, exportResult.MediaType); fileresult.FileDownloadName = exportResult.FileName; } else { fileresult = new FileContentResult(null, "application/octet-stream") { FileDownloadName = "NoData.csv" }; } return(fileresult); } catch (Exception ex) { Log.Error("ERROR IN EXPORT PROFILE GETDATA:", ex.Message); return(new FileContentResult(null, "application/octet-stream") { FileDownloadName = "NoData.csv" }); } }
public async Task CanHandleAttachmentExceptionsGracefully_Smuggler() { var backupPath = NewDataPath("BackupFolder"); var server = GetNewServer(); var allowDownload = false; var forwarder = new ProxyServer(8070, 8079) { VetoTransfer = (totalRead, buffer) => { var payload = System.Text.Encoding.UTF8.GetString(buffer.Array, buffer.Offset, buffer.Count); return payload.Contains("GET /static/users/678 ") && allowDownload == false; } }; try { string databaseName; using (var store = new DocumentStore { Url = "http://localhost:8079" }) { databaseName = store.DefaultDatabase; store.Initialize(); InsertAttachments(store, 2000); } var dumper = new SmugglerApi {SmugglerOptions = {Limit = 1500, Incremental = true}}; var allAttachments = new List<RavenJObject>(); ExportDataResult exportResult = null; try { exportResult = dumper.ExportData(new SmugglerExportOptions { ToFile = backupPath, From = new RavenConnectionStringOptions { Url = "http://localhost:8070", DefaultDatabase = databaseName, } }).Result; Assert.False(true, "Previous op should throw."); } catch (AggregateException e) { var inner = (SmugglerExportException)e.ExtractSingleInnerException(); exportResult = new ExportDataResult { FilePath = inner.File }; } allowDownload = true; using (var fileStream = new FileStream(exportResult.FilePath, FileMode.Open)) using (var stream = new GZipStream(fileStream, CompressionMode.Decompress)) { var chunk1 = RavenJToken.TryLoad(stream) as RavenJObject; var att1 = chunk1["Attachments"] as RavenJArray; allAttachments.AddRange(att1.Values<RavenJObject>()); } exportResult = await dumper.ExportData(new SmugglerExportOptions { ToFile = backupPath, From = new RavenConnectionStringOptions { Url = "http://localhost:8070", DefaultDatabase = databaseName, } }); using (var fileStream = new FileStream(exportResult.FilePath, FileMode.Open)) using (var stream = new GZipStream(fileStream, CompressionMode.Decompress)) { var chunk2 = RavenJToken.TryLoad(stream) as RavenJObject; var attr2 = chunk2["Attachments"] as RavenJArray; allAttachments.AddRange(attr2.Values<RavenJObject>()); } Assert.Equal(2000, allAttachments.Count()); } finally { IOExtensions.DeleteDirectory(backupPath); forwarder.Dispose(); server.Dispose(); } }
public async Task CanHandleDocumentExceptionsGracefully_Smuggler() { var backupPath = NewDataPath("BackupFolder"); var server = GetNewServer(databaseName: Constants.SystemDatabase); var alreadyReset = false; var port = 8070; var forwarder = new ProxyServer(ref port, 8079) { VetoTransfer = (totalRead, buffer) => { if (alreadyReset == false && totalRead > 25000) { alreadyReset = true; return(true); } return(false); } }; try { string databaseName; using (var store = new DocumentStore { Url = "http://localhost:8079" }) { databaseName = store.DefaultDatabase; store.Initialize(); InsertUsers(store, 0, 2000); } var dumper = new SmugglerDatabaseApi { Options = { Limit = 1900, Incremental = true } }; var allDocs = new List <RavenJObject>(); ExportDataResult exportResult = null; try { exportResult = await dumper.ExportData(new SmugglerExportOptions <RavenConnectionStringOptions> { ToFile = backupPath, From = new RavenConnectionStringOptions { Url = "http://localhost:" + port, DefaultDatabase = databaseName, } }); Assert.False(true, "Previous op should throw."); } catch (SmugglerExportException e) { exportResult = new ExportDataResult { FilePath = e.File }; } using (var fileSteam = new FileStream(exportResult.FilePath, FileMode.Open)) using (var stream = new GZipStream(fileSteam, CompressionMode.Decompress)) { var chunk1 = RavenJToken.TryLoad(stream) as RavenJObject; var doc1 = chunk1["Docs"] as RavenJArray; allDocs.AddRange(doc1.Values <RavenJObject>()); } exportResult = await dumper.ExportData(new SmugglerExportOptions <RavenConnectionStringOptions> { ToFile = backupPath, From = new RavenConnectionStringOptions { Url = "http://localhost:8070", DefaultDatabase = databaseName, } }); using (var fileStream = new FileStream(exportResult.FilePath, FileMode.Open)) using (var stream = new GZipStream(fileStream, CompressionMode.Decompress)) { var chunk2 = RavenJToken.TryLoad(stream) as RavenJObject; var doc2 = chunk2["Docs"] as RavenJArray; allDocs.AddRange(doc2.Values <RavenJObject>()); } Assert.Equal(2000, allDocs.Count(d => (d.Value <string>("Name") ?? String.Empty).StartsWith("User"))); } finally { forwarder.Dispose(); server.Dispose(); IOExtensions.DeleteDirectory(backupPath); } }
public void PurgeTombstones(ExportDataResult result) { throw new NotImplementedException("Purge tombstones is not supported for Command Line Smuggler"); }
public new void ExportDeletions(JsonTextWriter jsonWriter, SmugglerOptions options, ExportDataResult result, LastEtagsInfo maxEtags) { base.ExportDeletions(jsonWriter, options, result, maxEtags); }
public async Task DataDumperExportHandlesMaxEtagCorrectly() { using (var store = NewDocumentStore()) { using (var session = store.OpenSession()) { for (var i = 0; i < 10; i++) { session.Store(new User { Name = "oren #" + (i+1) }); } session.SaveChanges(); } using (var textStream = new StringWriter()) using (var writer = new JsonTextWriter(textStream)) { var dumper = new CustomDataDumper(store.SystemDatabase); var startEtag = store.SystemDatabase.Statistics.LastDocEtag.IncrementBy(-5); var endEtag = startEtag.IncrementBy(2); writer.WriteStartArray(); var lastEtag = await dumper.ExportDocuments(writer, startEtag, endEtag); writer.WriteEndArray(); writer.Flush(); // read exported content var exportedDocs = RavenJArray.Parse(textStream.GetStringBuilder().ToString()); Assert.Equal(2, exportedDocs.Count()); Assert.Equal("01000000-0000-0001-0000-000000000007", exportedDocs.First().Value<RavenJObject>("@metadata").Value<string>("@etag")); Assert.Equal("01000000-0000-0001-0000-000000000008", exportedDocs.Last().Value<RavenJObject>("@metadata").Value<string>("@etag")); Assert.Equal("01000000-0000-0001-0000-000000000008", lastEtag.ToString()); } using (var textStream = new StringWriter()) using (var writer = new JsonTextWriter(textStream)) { var dumper = new CustomDataDumper(store.SystemDatabase); var startEtag = store.SystemDatabase.Statistics.LastDocEtag.IncrementBy(-5); writer.WriteStartArray(); var lastEtag = await dumper.ExportDocuments(writer, startEtag, null); writer.WriteEndArray(); writer.Flush(); // read exported content var exportedDocs = RavenJArray.Parse(textStream.GetStringBuilder().ToString()); Assert.Equal(5, exportedDocs.Count()); Assert.Equal("01000000-0000-0001-0000-000000000007", exportedDocs.First().Value<RavenJObject>("@metadata").Value<string>("@etag")); Assert.Equal("01000000-0000-0001-0000-00000000000B", exportedDocs.Last().Value<RavenJObject>("@metadata").Value<string>("@etag")); Assert.Equal("01000000-0000-0001-0000-00000000000B", lastEtag.ToString()); } for (var i = 0; i < 10; i++) { store.DatabaseCommands.PutAttachment("attach/" + (i+1), null, new MemoryStream(new [] { (byte)i }), new RavenJObject()); } using (var textStream = new StringWriter()) using (var writer = new JsonTextWriter(textStream)) { var dumper = new CustomDataDumper(store.SystemDatabase); var startEtag = store.SystemDatabase.Statistics.LastAttachmentEtag.IncrementBy(-5); var endEtag = startEtag.IncrementBy(2); writer.WriteStartArray(); var lastEtag = await dumper.ExportAttachments(writer, startEtag, endEtag); writer.WriteEndArray(); writer.Flush(); // read exported content var exportedAttachments = RavenJArray.Parse(textStream.GetStringBuilder().ToString()); Assert.Equal(2, exportedAttachments.Count()); Assert.Equal("02000000-0000-0001-0000-000000000006", exportedAttachments.First().Value<string>("Etag")); Assert.Equal("02000000-0000-0001-0000-000000000007", exportedAttachments.Last().Value<string>("Etag")); Assert.Equal("02000000-0000-0001-0000-000000000007", lastEtag.ToString()); } using (var textStream = new StringWriter()) using (var writer = new JsonTextWriter(textStream)) { var dumper = new CustomDataDumper(store.SystemDatabase); var startEtag = store.SystemDatabase.Statistics.LastAttachmentEtag.IncrementBy(-5); writer.WriteStartArray(); var lastEtag = await dumper.ExportAttachments(writer, startEtag, null); writer.WriteEndArray(); writer.Flush(); // read exported content var exportedAttachments = RavenJArray.Parse(textStream.GetStringBuilder().ToString()); Assert.Equal(5, exportedAttachments.Count()); Assert.Equal("02000000-0000-0001-0000-000000000006", exportedAttachments.First().Value<string>("Etag")); Assert.Equal("02000000-0000-0001-0000-00000000000A", exportedAttachments.Last().Value<string>("Etag")); Assert.Equal("02000000-0000-0001-0000-00000000000A", lastEtag.ToString()); } WaitForIndexing(store); store.DatabaseCommands.DeleteByIndex("Raven/DocumentsByEntityName", new IndexQuery() { Query = "Tag:Users" }).WaitForCompletion(); for (var i = 0; i < 10; i++) { store.DatabaseCommands.DeleteAttachment("attach/" + (i+1), null); } Etag user6DeletionEtag = null, user9DeletionEtag = null, attach5DeletionEtag = null, attach7DeletionEtag = null; WaitForUserToContinueTheTest(store); store.SystemDatabase.TransactionalStorage.Batch(accessor => { user6DeletionEtag = accessor.Lists.Read(Constants.RavenPeriodicExportsDocsTombstones, "users/6").Etag; user9DeletionEtag = accessor.Lists.Read(Constants.RavenPeriodicExportsDocsTombstones, "users/9").Etag; attach5DeletionEtag = accessor.Lists.Read(Constants.RavenPeriodicExportsAttachmentsTombstones, "attach/5").Etag; attach7DeletionEtag = accessor.Lists.Read(Constants.RavenPeriodicExportsAttachmentsTombstones, "attach/7").Etag; }); using (var textStream = new StringWriter()) using (var writer = new JsonTextWriter(textStream)) { var dumper = new CustomDataDumper(store.SystemDatabase); writer.WriteStartObject(); var lastEtags = new LastEtagsInfo(); var exportResult = new ExportDataResult { LastDocDeleteEtag = user6DeletionEtag, LastAttachmentsDeleteEtag = attach5DeletionEtag }; lastEtags.LastDocDeleteEtag = user9DeletionEtag; lastEtags.LastAttachmentsDeleteEtag = attach7DeletionEtag; dumper.ExportDeletions(writer, exportResult, lastEtags).Wait(); writer.WriteEndObject(); writer.Flush(); // read exported content var exportJson = RavenJObject.Parse(textStream.GetStringBuilder().ToString()); var docsKeys = exportJson.Value<RavenJArray>("DocsDeletions").Select(x => x.Value<string>("Key")).ToArray(); var attachmentsKeys = exportJson.Value<RavenJArray>("AttachmentsDeletions") .Select(x => x.Value<string>("Key")) .ToArray(); Assert.Equal(new [] { "users/7", "users/8", "users/9" }, docsKeys); Assert.Equal(new [] { "attach/6", "attach/7" }, attachmentsKeys); } } }
public override Task ExportDeletions(JsonTextWriter jsonWriter, ExportDataResult result, LastEtagsInfo maxEtags) { return base.ExportDeletions(jsonWriter, result, maxEtags); }
public void CanProperlyReadLastEtagUsingPreviousFormat() { var backupPath = NewDataPath("BackupFolder", forceCreateDir:true); var etagFileLocation = Path.Combine(Path.GetDirectoryName(backupPath), "IncrementalExport.state.json"); using (var streamWriter = new StreamWriter(File.Create(etagFileLocation))) { new RavenJObject { {"LastDocEtag", Etag.Parse("00000000-0000-0000-0000-000000000001").ToString()}, {"LastAttachmentEtag", Etag.Parse("00000000-0000-0000-0000-000000000002").ToString()} }.WriteTo(new JsonTextWriter(streamWriter)); streamWriter.Flush(); } var result = new ExportDataResult { FilePath = backupPath }; SmugglerDatabaseApiBase.ReadLastEtagsFromFile(result); Assert.Equal("00000000-0000-0000-0000-000000000001", result.LastDocsEtag.ToString()); Assert.Equal("00000000-0000-0000-0000-000000000002", result.LastAttachmentsEtag.ToString()); Assert.Equal(Etag.Empty, result.LastDocDeleteEtag); Assert.Equal(Etag.Empty, result.LastAttachmentsDeleteEtag); }
public override Task ExportDeletions(JsonTextWriter jsonWriter, ExportDataResult result, LastEtagsInfo maxEtags) { return(base.ExportDeletions(jsonWriter, result, maxEtags)); }
public async Task CanHandleAttachmentExceptionsGracefully_Smuggler() { var backupPath = NewDataPath("BackupFolder"); var server = GetNewServer(); var allowDownload = false; var port = 8070; var forwarder = new ProxyServer(ref port, 8079) { VetoTransfer = (totalRead, buffer) => { var payload = System.Text.Encoding.UTF8.GetString(buffer.Array, buffer.Offset, buffer.Count); return(payload.Contains("GET /static/users/678 ") && allowDownload == false); } }; try { string databaseName; using (var store = new DocumentStore { Url = "http://localhost:8079" }) { databaseName = store.DefaultDatabase; store.Initialize(); InsertAttachments(store, 2000); } var dumper = new SmugglerDatabaseApi { Options = { Limit = 1500, Incremental = true } }; var allAttachments = new List <RavenJObject>(); ExportDataResult exportResult = null; try { exportResult = dumper.ExportData(new SmugglerExportOptions <RavenConnectionStringOptions> { ToFile = backupPath, From = new RavenConnectionStringOptions { Url = "http://localhost:" + port, DefaultDatabase = databaseName, } }).Result; Assert.False(true, "Previous op should throw."); } catch (AggregateException e) { var inner = (SmugglerExportException)e.ExtractSingleInnerException(); exportResult = new ExportDataResult { FilePath = inner.File }; } allowDownload = true; using (var fileStream = new FileStream(exportResult.FilePath, FileMode.Open)) using (var stream = new GZipStream(fileStream, CompressionMode.Decompress)) { var chunk1 = RavenJToken.TryLoad(stream) as RavenJObject; var att1 = chunk1["Attachments"] as RavenJArray; allAttachments.AddRange(att1.Values <RavenJObject>()); } exportResult = await dumper.ExportData(new SmugglerExportOptions <RavenConnectionStringOptions> { ToFile = backupPath, From = new RavenConnectionStringOptions { Url = "http://localhost:8070", DefaultDatabase = databaseName, } }); using (var fileStream = new FileStream(exportResult.FilePath, FileMode.Open)) using (var stream = new GZipStream(fileStream, CompressionMode.Decompress)) { var chunk2 = RavenJToken.TryLoad(stream) as RavenJObject; var attr2 = chunk2["Attachments"] as RavenJArray; allAttachments.AddRange(attr2.Values <RavenJObject>()); } Assert.Equal(2000, allAttachments.Count()); } finally { IOExtensions.DeleteDirectory(backupPath); forwarder.Dispose(); server.Dispose(); } }
public async Task DataDumperExportHandlesMaxEtagCorrectly() { using (var store = NewDocumentStore()) { using (var session = store.OpenSession()) { for (var i = 0; i < 10; i++) { session.Store(new User { Name = "oren #" + (i + 1) }); } session.SaveChanges(); } using (var textStream = new StringWriter()) using (var writer = new JsonTextWriter(textStream)) { var dumper = new CustomDataDumper(store.SystemDatabase); var startEtag = store.SystemDatabase.Statistics.LastDocEtag.IncrementBy(-5); var endEtag = startEtag.IncrementBy(2); writer.WriteStartArray(); var lastEtag = await dumper.ExportDocuments(writer, startEtag, endEtag); writer.WriteEndArray(); writer.Flush(); // read exported content var exportedDocs = RavenJArray.Parse(textStream.GetStringBuilder().ToString()); Assert.Equal(2, exportedDocs.Count()); Assert.Equal("01000000-0000-0001-0000-000000000007", exportedDocs.First().Value <RavenJObject>("@metadata").Value <string>("@etag")); Assert.Equal("01000000-0000-0001-0000-000000000008", exportedDocs.Last().Value <RavenJObject>("@metadata").Value <string>("@etag")); Assert.Equal("01000000-0000-0001-0000-000000000008", lastEtag.ToString()); } using (var textStream = new StringWriter()) using (var writer = new JsonTextWriter(textStream)) { var dumper = new CustomDataDumper(store.SystemDatabase); var startEtag = store.SystemDatabase.Statistics.LastDocEtag.IncrementBy(-5); writer.WriteStartArray(); var lastEtag = await dumper.ExportDocuments(writer, startEtag, null); writer.WriteEndArray(); writer.Flush(); // read exported content var exportedDocs = RavenJArray.Parse(textStream.GetStringBuilder().ToString()); Assert.Equal(5, exportedDocs.Count()); Assert.Equal("01000000-0000-0001-0000-000000000007", exportedDocs.First().Value <RavenJObject>("@metadata").Value <string>("@etag")); Assert.Equal("01000000-0000-0001-0000-00000000000B", exportedDocs.Last().Value <RavenJObject>("@metadata").Value <string>("@etag")); Assert.Equal("01000000-0000-0001-0000-00000000000B", lastEtag.ToString()); } for (var i = 0; i < 10; i++) { store.DatabaseCommands.PutAttachment("attach/" + (i + 1), null, new MemoryStream(new [] { (byte)i }), new RavenJObject()); } using (var textStream = new StringWriter()) using (var writer = new JsonTextWriter(textStream)) { var dumper = new CustomDataDumper(store.SystemDatabase); var startEtag = store.SystemDatabase.Statistics.LastAttachmentEtag.IncrementBy(-5); var endEtag = startEtag.IncrementBy(2); writer.WriteStartArray(); var lastEtag = await dumper.ExportAttachments(writer, startEtag, endEtag); writer.WriteEndArray(); writer.Flush(); // read exported content var exportedAttachments = RavenJArray.Parse(textStream.GetStringBuilder().ToString()); Assert.Equal(2, exportedAttachments.Count()); Assert.Equal("02000000-0000-0001-0000-000000000006", exportedAttachments.First().Value <string>("Etag")); Assert.Equal("02000000-0000-0001-0000-000000000007", exportedAttachments.Last().Value <string>("Etag")); Assert.Equal("02000000-0000-0001-0000-000000000007", lastEtag.ToString()); } using (var textStream = new StringWriter()) using (var writer = new JsonTextWriter(textStream)) { var dumper = new CustomDataDumper(store.SystemDatabase); var startEtag = store.SystemDatabase.Statistics.LastAttachmentEtag.IncrementBy(-5); writer.WriteStartArray(); var lastEtag = await dumper.ExportAttachments(writer, startEtag, null); writer.WriteEndArray(); writer.Flush(); // read exported content var exportedAttachments = RavenJArray.Parse(textStream.GetStringBuilder().ToString()); Assert.Equal(5, exportedAttachments.Count()); Assert.Equal("02000000-0000-0001-0000-000000000006", exportedAttachments.First().Value <string>("Etag")); Assert.Equal("02000000-0000-0001-0000-00000000000A", exportedAttachments.Last().Value <string>("Etag")); Assert.Equal("02000000-0000-0001-0000-00000000000A", lastEtag.ToString()); } WaitForIndexing(store); store.DatabaseCommands.DeleteByIndex("Raven/DocumentsByEntityName", new IndexQuery() { Query = "Tag:Users" }).WaitForCompletion(); for (var i = 0; i < 10; i++) { store.DatabaseCommands.DeleteAttachment("attach/" + (i + 1), null); } Etag user6DeletionEtag = null, user9DeletionEtag = null, attach5DeletionEtag = null, attach7DeletionEtag = null; WaitForUserToContinueTheTest(store); store.SystemDatabase.TransactionalStorage.Batch(accessor => { user6DeletionEtag = accessor.Lists.Read(Constants.RavenPeriodicExportsDocsTombstones, "users/6").Etag; user9DeletionEtag = accessor.Lists.Read(Constants.RavenPeriodicExportsDocsTombstones, "users/9").Etag; attach5DeletionEtag = accessor.Lists.Read(Constants.RavenPeriodicExportsAttachmentsTombstones, "attach/5").Etag; attach7DeletionEtag = accessor.Lists.Read(Constants.RavenPeriodicExportsAttachmentsTombstones, "attach/7").Etag; }); using (var textStream = new StringWriter()) using (var writer = new JsonTextWriter(textStream)) { var dumper = new CustomDataDumper(store.SystemDatabase); writer.WriteStartObject(); var lastEtags = new LastEtagsInfo(); var exportResult = new ExportDataResult { LastDocDeleteEtag = user6DeletionEtag, LastAttachmentsDeleteEtag = attach5DeletionEtag }; lastEtags.LastDocDeleteEtag = user9DeletionEtag; lastEtags.LastAttachmentsDeleteEtag = attach7DeletionEtag; dumper.ExportDeletions(writer, exportResult, lastEtags).Wait(); writer.WriteEndObject(); writer.Flush(); // read exported content var exportJson = RavenJObject.Parse(textStream.GetStringBuilder().ToString()); var docsKeys = exportJson.Value <RavenJArray>("DocsDeletions").Select(x => x.Value <string>("Key")).ToArray(); var attachmentsKeys = exportJson.Value <RavenJArray>("AttachmentsDeletions") .Select(x => x.Value <string>("Key")) .ToArray(); Assert.Equal(new [] { "users/7", "users/8", "users/9" }, docsKeys); Assert.Equal(new [] { "attach/6", "attach/7" }, attachmentsKeys); } } }
public override Task ExportDeletions(JsonTextWriter jsonWriter, ExportDataResult result, LastEtagsInfo maxEtagsToFetch) { throw new NotSupportedException("Exporting deletions is not supported for Command Line Smuggler"); }
protected override void ExportDeletions(JsonTextWriter jsonWriter, SmugglerOptions options, ExportDataResult result, LastEtagsInfo maxEtagsToFetch) { throw new NotImplementedException("Export Deletions is not supported for command line smuggler"); }
public static void ReadLastEtagsFromFile(ExportDataResult result) { var log = LogManager.GetCurrentClassLogger(); var etagFileLocation = Path.Combine(result.FilePath, IncrementalExportStateFile); if (!File.Exists(etagFileLocation)) return; using (var streamReader = new StreamReader(new FileStream(etagFileLocation, FileMode.Open))) using (var jsonReader = new JsonTextReader(streamReader)) { RavenJObject ravenJObject; try { ravenJObject = RavenJObject.Load(jsonReader); } catch (Exception e) { log.WarnException("Could not parse etag document from file : " + etagFileLocation + ", ignoring, will start from scratch", e); return; } result.LastDocsEtag = Etag.Parse(ravenJObject.Value<string>("LastDocEtag")); result.LastAttachmentsEtag = Etag.Parse(ravenJObject.Value<string>("LastAttachmentEtag")); result.LastDocDeleteEtag = Etag.Parse(ravenJObject.Value<string>("LastDocDeleteEtag") ?? Etag.Empty.ToString()); result.LastAttachmentsDeleteEtag = Etag.Parse(ravenJObject.Value<string>("LastAttachmentsDeleteEtag") ?? Etag.Empty.ToString()); } }
public static void WriteLastEtagsToFile(ExportDataResult result, string backupPath) { // ReSharper disable once AssignNullToNotNullAttribute var etagFileLocation = Path.Combine(Path.GetDirectoryName(backupPath), IncrementalExportStateFile); using (var streamWriter = new StreamWriter(File.Create(etagFileLocation))) { new RavenJObject { {"LastDocEtag", result.LastDocsEtag.ToString()}, {"LastAttachmentEtag", result.LastAttachmentsEtag.ToString()}, {"LastDocDeleteEtag", result.LastDocDeleteEtag.ToString()}, {"LastAttachmentsDeleteEtag", result.LastAttachmentsDeleteEtag.ToString()} }.WriteTo(new JsonTextWriter(streamWriter)); streamWriter.Flush(); } }
/// <summary> /// /// </summary> /// <param name="jsonWriter"></param> /// <param name="options"></param> /// <param name="result"></param> /// <param name="maxEtags">Max etags are inclusive</param> protected async override void ExportDeletions(JsonTextWriter jsonWriter, SmugglerOptions options, ExportDataResult result, LastEtagsInfo maxEtags) { jsonWriter.WritePropertyName("DocsDeletions"); jsonWriter.WriteStartArray(); result.LastDocDeleteEtag = await ExportDocumentsDeletion(options, jsonWriter, result.LastDocDeleteEtag, maxEtags.LastDocDeleteEtag.IncrementBy(1)); jsonWriter.WriteEndArray(); jsonWriter.WritePropertyName("AttachmentsDeletions"); jsonWriter.WriteStartArray(); result.LastAttachmentsDeleteEtag = await ExportAttachmentsDeletion(options, jsonWriter, result.LastAttachmentsDeleteEtag, maxEtags.LastAttachmentsDeleteEtag.IncrementBy(1)); jsonWriter.WriteEndArray(); }