public async Task DataDumperExportHandlesMaxEtagCorrectly() { using (var store = NewDocumentStore()) { using (var session = store.OpenSession()) { for (var i = 0; i < 10; i++) { session.Store(new User { Name = "oren #" + (i + 1) }); } session.SaveChanges(); } using (var textStream = new StringWriter()) using (var writer = new JsonTextWriter(textStream)) { var dumper = new CustomDataDumper(store.SystemDatabase); var startEtag = store.SystemDatabase.Statistics.LastDocEtag.IncrementBy(-5); var endEtag = startEtag.IncrementBy(2); writer.WriteStartArray(); var lastEtag = await dumper.ExportDocuments(writer, startEtag, endEtag); writer.WriteEndArray(); writer.Flush(); // read exported content var exportedDocs = RavenJArray.Parse(textStream.GetStringBuilder().ToString()); Assert.Equal(2, exportedDocs.Count()); Assert.Equal("01000000-0000-0001-0000-000000000007", exportedDocs.First().Value<RavenJObject>("@metadata").Value<string>("@etag")); Assert.Equal("01000000-0000-0001-0000-000000000008", exportedDocs.Last().Value<RavenJObject>("@metadata").Value<string>("@etag")); Assert.Equal("01000000-0000-0001-0000-000000000008", lastEtag.ToString()); } using (var textStream = new StringWriter()) using (var writer = new JsonTextWriter(textStream)) { var dumper = new CustomDataDumper(store.SystemDatabase); var startEtag = store.SystemDatabase.Statistics.LastDocEtag.IncrementBy(-5); writer.WriteStartArray(); var lastEtag = await dumper.ExportDocuments(writer, startEtag, null); writer.WriteEndArray(); writer.Flush(); // read exported content var exportedDocs = RavenJArray.Parse(textStream.GetStringBuilder().ToString()); Assert.Equal(5, exportedDocs.Count()); Assert.Equal("01000000-0000-0001-0000-000000000007", exportedDocs.First().Value<RavenJObject>("@metadata").Value<string>("@etag")); Assert.Equal("01000000-0000-0001-0000-00000000000B", exportedDocs.Last().Value<RavenJObject>("@metadata").Value<string>("@etag")); Assert.Equal("01000000-0000-0001-0000-00000000000B", lastEtag.ToString()); } for (var i = 0; i < 10; i++) { store.DatabaseCommands.PutAttachment("attach/" + (i + 1), null, new MemoryStream(new[] { (byte)i }), new RavenJObject()); } using (var textStream = new StringWriter()) using (var writer = new JsonTextWriter(textStream)) { var dumper = new CustomDataDumper(store.SystemDatabase); var startEtag = store.SystemDatabase.Statistics.LastAttachmentEtag.IncrementBy(-5); var endEtag = startEtag.IncrementBy(2); writer.WriteStartArray(); var lastEtag = await dumper.ExportAttachments(writer, startEtag, endEtag); writer.WriteEndArray(); writer.Flush(); // read exported content var exportedAttachments = RavenJArray.Parse(textStream.GetStringBuilder().ToString()); Assert.Equal(2, exportedAttachments.Count()); Assert.Equal("02000000-0000-0001-0000-000000000006", exportedAttachments.First().Value<string>("Etag")); Assert.Equal("02000000-0000-0001-0000-000000000007", exportedAttachments.Last().Value<string>("Etag")); Assert.Equal("02000000-0000-0001-0000-000000000007", lastEtag.ToString()); } using (var textStream = new StringWriter()) using (var writer = new JsonTextWriter(textStream)) { var dumper = new CustomDataDumper(store.SystemDatabase); var startEtag = store.SystemDatabase.Statistics.LastAttachmentEtag.IncrementBy(-5); writer.WriteStartArray(); var lastEtag = await dumper.ExportAttachments(writer, startEtag, null); writer.WriteEndArray(); writer.Flush(); // read exported content var exportedAttachments = RavenJArray.Parse(textStream.GetStringBuilder().ToString()); Assert.Equal(5, exportedAttachments.Count()); Assert.Equal("02000000-0000-0001-0000-000000000006", exportedAttachments.First().Value<string>("Etag")); Assert.Equal("02000000-0000-0001-0000-00000000000A", exportedAttachments.Last().Value<string>("Etag")); Assert.Equal("02000000-0000-0001-0000-00000000000A", lastEtag.ToString()); } WaitForIndexing(store); store.DatabaseCommands.DeleteByIndex("Raven/DocumentsByEntityName", new IndexQuery() { Query = "Tag:Users" }).WaitForCompletion(); for (var i = 0; i < 10; i++) { store.DatabaseCommands.DeleteAttachment("attach/" + (i + 1), null); } Etag user6DeletionEtag = null, user9DeletionEtag = null, attach5DeletionEtag = null, attach7DeletionEtag = null; WaitForUserToContinueTheTest(store); store.SystemDatabase.TransactionalStorage.Batch(accessor => { user6DeletionEtag = accessor.Lists.Read(Constants.RavenPeriodicExportsDocsTombstones, "users/6").Etag; user9DeletionEtag = accessor.Lists.Read(Constants.RavenPeriodicExportsDocsTombstones, "users/9").Etag; attach5DeletionEtag = accessor.Lists.Read(Constants.RavenPeriodicExportsAttachmentsTombstones, "attach/5").Etag; attach7DeletionEtag = accessor.Lists.Read(Constants.RavenPeriodicExportsAttachmentsTombstones, "attach/7").Etag; }); using (var textStream = new StringWriter()) using (var writer = new JsonTextWriter(textStream)) { var dumper = new CustomDataDumper(store.SystemDatabase); writer.WriteStartObject(); var lastEtags = new LastEtagsInfo(); var exportResult = new OperationState { LastDocDeleteEtag = user6DeletionEtag, LastAttachmentsDeleteEtag = attach5DeletionEtag }; lastEtags.LastDocDeleteEtag = user9DeletionEtag; lastEtags.LastAttachmentsDeleteEtag = attach7DeletionEtag; dumper.ExportDeletions(writer, exportResult, lastEtags).Wait(); writer.WriteEndObject(); writer.Flush(); // read exported content var exportJson = RavenJObject.Parse(textStream.GetStringBuilder().ToString()); var docsKeys = exportJson.Value<RavenJArray>("DocsDeletions").Select(x => x.Value<string>("Key")).ToArray(); var attachmentsKeys = exportJson.Value<RavenJArray>("AttachmentsDeletions") .Select(x => x.Value<string>("Key")) .ToArray(); Assert.Equal(new[] { "users/7", "users/8", "users/9" }, docsKeys); Assert.Equal(new[] { "attach/6", "attach/7" }, attachmentsKeys); } } }
public async Task DataDumperExportHandlesMaxEtagCorrectly() { using (var store = NewDocumentStore()) { using (var session = store.OpenSession()) { for (var i = 0; i < 10; i++) { session.Store(new User { Name = "oren #" + (i + 1) }); } session.SaveChanges(); } using (var textStream = new StringWriter()) using (var writer = new JsonTextWriter(textStream)) { var dumper = new CustomDataDumper(store.SystemDatabase); var startEtag = store.SystemDatabase.Statistics.LastDocEtag.IncrementBy(-5); var endEtag = startEtag.IncrementBy(2); writer.WriteStartArray(); var lastEtag = await dumper.ExportDocuments(writer, startEtag, endEtag); writer.WriteEndArray(); writer.Flush(); // read exported content var exportedDocs = RavenJArray.Parse(textStream.GetStringBuilder().ToString()); Assert.Equal(2, exportedDocs.Count()); Assert.Equal("01000000-0000-0001-0000-000000000007", exportedDocs.First().Value <RavenJObject>("@metadata").Value <string>("@etag")); Assert.Equal("01000000-0000-0001-0000-000000000008", exportedDocs.Last().Value <RavenJObject>("@metadata").Value <string>("@etag")); Assert.Equal("01000000-0000-0001-0000-000000000008", lastEtag.ToString()); } using (var textStream = new StringWriter()) using (var writer = new JsonTextWriter(textStream)) { var dumper = new CustomDataDumper(store.SystemDatabase); var startEtag = store.SystemDatabase.Statistics.LastDocEtag.IncrementBy(-5); writer.WriteStartArray(); var lastEtag = await dumper.ExportDocuments(writer, startEtag, null); writer.WriteEndArray(); writer.Flush(); // read exported content var exportedDocs = RavenJArray.Parse(textStream.GetStringBuilder().ToString()); Assert.Equal(5, exportedDocs.Count()); Assert.Equal("01000000-0000-0001-0000-000000000007", exportedDocs.First().Value <RavenJObject>("@metadata").Value <string>("@etag")); Assert.Equal("01000000-0000-0001-0000-00000000000B", exportedDocs.Last().Value <RavenJObject>("@metadata").Value <string>("@etag")); Assert.Equal("01000000-0000-0001-0000-00000000000B", lastEtag.ToString()); } for (var i = 0; i < 10; i++) { store.DatabaseCommands.PutAttachment("attach/" + (i + 1), null, new MemoryStream(new [] { (byte)i }), new RavenJObject()); } using (var textStream = new StringWriter()) using (var writer = new JsonTextWriter(textStream)) { var dumper = new CustomDataDumper(store.SystemDatabase); var startEtag = store.SystemDatabase.Statistics.LastAttachmentEtag.IncrementBy(-5); var endEtag = startEtag.IncrementBy(2); writer.WriteStartArray(); var lastEtag = await dumper.ExportAttachments(writer, startEtag, endEtag); writer.WriteEndArray(); writer.Flush(); // read exported content var exportedAttachments = RavenJArray.Parse(textStream.GetStringBuilder().ToString()); Assert.Equal(2, exportedAttachments.Count()); Assert.Equal("02000000-0000-0001-0000-000000000006", exportedAttachments.First().Value <string>("Etag")); Assert.Equal("02000000-0000-0001-0000-000000000007", exportedAttachments.Last().Value <string>("Etag")); Assert.Equal("02000000-0000-0001-0000-000000000007", lastEtag.ToString()); } using (var textStream = new StringWriter()) using (var writer = new JsonTextWriter(textStream)) { var dumper = new CustomDataDumper(store.SystemDatabase); var startEtag = store.SystemDatabase.Statistics.LastAttachmentEtag.IncrementBy(-5); writer.WriteStartArray(); var lastEtag = await dumper.ExportAttachments(writer, startEtag, null); writer.WriteEndArray(); writer.Flush(); // read exported content var exportedAttachments = RavenJArray.Parse(textStream.GetStringBuilder().ToString()); Assert.Equal(5, exportedAttachments.Count()); Assert.Equal("02000000-0000-0001-0000-000000000006", exportedAttachments.First().Value <string>("Etag")); Assert.Equal("02000000-0000-0001-0000-00000000000A", exportedAttachments.Last().Value <string>("Etag")); Assert.Equal("02000000-0000-0001-0000-00000000000A", lastEtag.ToString()); } WaitForIndexing(store); store.DatabaseCommands.DeleteByIndex("Raven/DocumentsByEntityName", new IndexQuery() { Query = "Tag:Users" }).WaitForCompletion(); for (var i = 0; i < 10; i++) { store.DatabaseCommands.DeleteAttachment("attach/" + (i + 1), null); } Etag user6DeletionEtag = null, user9DeletionEtag = null, attach5DeletionEtag = null, attach7DeletionEtag = null; WaitForUserToContinueTheTest(store); store.SystemDatabase.TransactionalStorage.Batch(accessor => { user6DeletionEtag = accessor.Lists.Read(Constants.RavenPeriodicExportsDocsTombstones, "users/6").Etag; user9DeletionEtag = accessor.Lists.Read(Constants.RavenPeriodicExportsDocsTombstones, "users/9").Etag; attach5DeletionEtag = accessor.Lists.Read(Constants.RavenPeriodicExportsAttachmentsTombstones, "attach/5").Etag; attach7DeletionEtag = accessor.Lists.Read(Constants.RavenPeriodicExportsAttachmentsTombstones, "attach/7").Etag; }); using (var textStream = new StringWriter()) using (var writer = new JsonTextWriter(textStream)) { var dumper = new CustomDataDumper(store.SystemDatabase); writer.WriteStartObject(); var lastEtags = new LastEtagsInfo(); var exportResult = new ExportDataResult { LastDocDeleteEtag = user6DeletionEtag, LastAttachmentsDeleteEtag = attach5DeletionEtag }; lastEtags.LastDocDeleteEtag = user9DeletionEtag; lastEtags.LastAttachmentsDeleteEtag = attach7DeletionEtag; dumper.ExportDeletions(writer, exportResult, lastEtags).Wait(); writer.WriteEndObject(); writer.Flush(); // read exported content var exportJson = RavenJObject.Parse(textStream.GetStringBuilder().ToString()); var docsKeys = exportJson.Value <RavenJArray>("DocsDeletions").Select(x => x.Value <string>("Key")).ToArray(); var attachmentsKeys = exportJson.Value <RavenJArray>("AttachmentsDeletions") .Select(x => x.Value <string>("Key")) .ToArray(); Assert.Equal(new [] { "users/7", "users/8", "users/9" }, docsKeys); Assert.Equal(new [] { "attach/6", "attach/7" }, attachmentsKeys); } } }