public async Task DataDumperExportHandlesMaxEtagCorrectly()
{
using (var store = NewDocumentStore())
{
using (var session = store.OpenSession())
{
for (var i = 0; i < 10; i++)
{
session.Store(new User { Name = "oren #" + (i + 1) });
}
session.SaveChanges();
}
using (var textStream = new StringWriter())
using (var writer = new JsonTextWriter(textStream))
{
var dumper = new CustomDataDumper(store.SystemDatabase);
var startEtag = store.SystemDatabase.Statistics.LastDocEtag.IncrementBy(-5);
var endEtag = startEtag.IncrementBy(2);
writer.WriteStartArray();
var lastEtag = await dumper.ExportDocuments(writer, startEtag, endEtag);
writer.WriteEndArray();
writer.Flush();
// read exported content
var exportedDocs = RavenJArray.Parse(textStream.GetStringBuilder().ToString());
Assert.Equal(2, exportedDocs.Count());
Assert.Equal("01000000-0000-0001-0000-000000000007", exportedDocs.First().Value<RavenJObject>("@metadata").Value<string>("@etag"));
Assert.Equal("01000000-0000-0001-0000-000000000008", exportedDocs.Last().Value<RavenJObject>("@metadata").Value<string>("@etag"));
Assert.Equal("01000000-0000-0001-0000-000000000008", lastEtag.ToString());
}
using (var textStream = new StringWriter())
using (var writer = new JsonTextWriter(textStream))
{
var dumper = new CustomDataDumper(store.SystemDatabase);
var startEtag = store.SystemDatabase.Statistics.LastDocEtag.IncrementBy(-5);
writer.WriteStartArray();
var lastEtag = await dumper.ExportDocuments(writer, startEtag, null);
writer.WriteEndArray();
writer.Flush();
// read exported content
var exportedDocs = RavenJArray.Parse(textStream.GetStringBuilder().ToString());
Assert.Equal(5, exportedDocs.Count());
Assert.Equal("01000000-0000-0001-0000-000000000007", exportedDocs.First().Value<RavenJObject>("@metadata").Value<string>("@etag"));
Assert.Equal("01000000-0000-0001-0000-00000000000B", exportedDocs.Last().Value<RavenJObject>("@metadata").Value<string>("@etag"));
Assert.Equal("01000000-0000-0001-0000-00000000000B", lastEtag.ToString());
}
for (var i = 0; i < 10; i++)
{
store.DatabaseCommands.PutAttachment("attach/" + (i + 1), null, new MemoryStream(new[] { (byte)i }), new RavenJObject());
}
using (var textStream = new StringWriter())
using (var writer = new JsonTextWriter(textStream))
{
var dumper = new CustomDataDumper(store.SystemDatabase);
var startEtag = store.SystemDatabase.Statistics.LastAttachmentEtag.IncrementBy(-5);
var endEtag = startEtag.IncrementBy(2);
writer.WriteStartArray();
var lastEtag = await dumper.ExportAttachments(writer, startEtag, endEtag);
writer.WriteEndArray();
writer.Flush();
// read exported content
var exportedAttachments = RavenJArray.Parse(textStream.GetStringBuilder().ToString());
Assert.Equal(2, exportedAttachments.Count());
Assert.Equal("02000000-0000-0001-0000-000000000006", exportedAttachments.First().Value<string>("Etag"));
Assert.Equal("02000000-0000-0001-0000-000000000007", exportedAttachments.Last().Value<string>("Etag"));
Assert.Equal("02000000-0000-0001-0000-000000000007", lastEtag.ToString());
}
using (var textStream = new StringWriter())
using (var writer = new JsonTextWriter(textStream))
{
var dumper = new CustomDataDumper(store.SystemDatabase);
var startEtag = store.SystemDatabase.Statistics.LastAttachmentEtag.IncrementBy(-5);
writer.WriteStartArray();
var lastEtag = await dumper.ExportAttachments(writer, startEtag, null);
writer.WriteEndArray();
writer.Flush();
// read exported content
var exportedAttachments = RavenJArray.Parse(textStream.GetStringBuilder().ToString());
Assert.Equal(5, exportedAttachments.Count());
Assert.Equal("02000000-0000-0001-0000-000000000006", exportedAttachments.First().Value<string>("Etag"));
Assert.Equal("02000000-0000-0001-0000-00000000000A", exportedAttachments.Last().Value<string>("Etag"));
Assert.Equal("02000000-0000-0001-0000-00000000000A", lastEtag.ToString());
}
WaitForIndexing(store);
store.DatabaseCommands.DeleteByIndex("Raven/DocumentsByEntityName", new IndexQuery()
{
Query = "Tag:Users"
}).WaitForCompletion();
for (var i = 0; i < 10; i++)
{
store.DatabaseCommands.DeleteAttachment("attach/" + (i + 1), null);
}
Etag user6DeletionEtag = null, user9DeletionEtag = null, attach5DeletionEtag = null, attach7DeletionEtag = null;
WaitForUserToContinueTheTest(store);
store.SystemDatabase.TransactionalStorage.Batch(accessor =>
{
user6DeletionEtag =
accessor.Lists.Read(Constants.RavenPeriodicExportsDocsTombstones, "users/6").Etag;
user9DeletionEtag =
accessor.Lists.Read(Constants.RavenPeriodicExportsDocsTombstones, "users/9").Etag;
attach5DeletionEtag =
accessor.Lists.Read(Constants.RavenPeriodicExportsAttachmentsTombstones, "attach/5").Etag;
attach7DeletionEtag =
accessor.Lists.Read(Constants.RavenPeriodicExportsAttachmentsTombstones, "attach/7").Etag;
});
using (var textStream = new StringWriter())
using (var writer = new JsonTextWriter(textStream))
{
var dumper = new CustomDataDumper(store.SystemDatabase);
writer.WriteStartObject();
var lastEtags = new LastEtagsInfo();
var exportResult = new OperationState
{
LastDocDeleteEtag = user6DeletionEtag,
LastAttachmentsDeleteEtag = attach5DeletionEtag
};
lastEtags.LastDocDeleteEtag = user9DeletionEtag;
lastEtags.LastAttachmentsDeleteEtag = attach7DeletionEtag;
dumper.ExportDeletions(writer, exportResult, lastEtags).Wait();
writer.WriteEndObject();
writer.Flush();
// read exported content
var exportJson = RavenJObject.Parse(textStream.GetStringBuilder().ToString());
var docsKeys =
exportJson.Value<RavenJArray>("DocsDeletions").Select(x => x.Value<string>("Key")).ToArray();
var attachmentsKeys =
exportJson.Value<RavenJArray>("AttachmentsDeletions")
.Select(x => x.Value<string>("Key"))
.ToArray();
Assert.Equal(new[] { "users/7", "users/8", "users/9" }, docsKeys);
Assert.Equal(new[] { "attach/6", "attach/7" }, attachmentsKeys);
}
}
}
public async Task DataDumperExportHandlesMaxEtagCorrectly()
{
using (var store = NewDocumentStore())
{
using (var session = store.OpenSession())
{
for (var i = 0; i < 10; i++)
{
session.Store(new User {
Name = "oren #" + (i + 1)
});
}
session.SaveChanges();
}
using (var textStream = new StringWriter())
using (var writer = new JsonTextWriter(textStream))
{
var dumper = new CustomDataDumper(store.SystemDatabase);
var startEtag = store.SystemDatabase.Statistics.LastDocEtag.IncrementBy(-5);
var endEtag = startEtag.IncrementBy(2);
writer.WriteStartArray();
var lastEtag = await dumper.ExportDocuments(writer, startEtag, endEtag);
writer.WriteEndArray();
writer.Flush();
// read exported content
var exportedDocs = RavenJArray.Parse(textStream.GetStringBuilder().ToString());
Assert.Equal(2, exportedDocs.Count());
Assert.Equal("01000000-0000-0001-0000-000000000007", exportedDocs.First().Value <RavenJObject>("@metadata").Value <string>("@etag"));
Assert.Equal("01000000-0000-0001-0000-000000000008", exportedDocs.Last().Value <RavenJObject>("@metadata").Value <string>("@etag"));
Assert.Equal("01000000-0000-0001-0000-000000000008", lastEtag.ToString());
}
using (var textStream = new StringWriter())
using (var writer = new JsonTextWriter(textStream))
{
var dumper = new CustomDataDumper(store.SystemDatabase);
var startEtag = store.SystemDatabase.Statistics.LastDocEtag.IncrementBy(-5);
writer.WriteStartArray();
var lastEtag = await dumper.ExportDocuments(writer, startEtag, null);
writer.WriteEndArray();
writer.Flush();
// read exported content
var exportedDocs = RavenJArray.Parse(textStream.GetStringBuilder().ToString());
Assert.Equal(5, exportedDocs.Count());
Assert.Equal("01000000-0000-0001-0000-000000000007", exportedDocs.First().Value <RavenJObject>("@metadata").Value <string>("@etag"));
Assert.Equal("01000000-0000-0001-0000-00000000000B", exportedDocs.Last().Value <RavenJObject>("@metadata").Value <string>("@etag"));
Assert.Equal("01000000-0000-0001-0000-00000000000B", lastEtag.ToString());
}
for (var i = 0; i < 10; i++)
{
store.DatabaseCommands.PutAttachment("attach/" + (i + 1), null, new MemoryStream(new [] { (byte)i }), new RavenJObject());
}
using (var textStream = new StringWriter())
using (var writer = new JsonTextWriter(textStream))
{
var dumper = new CustomDataDumper(store.SystemDatabase);
var startEtag = store.SystemDatabase.Statistics.LastAttachmentEtag.IncrementBy(-5);
var endEtag = startEtag.IncrementBy(2);
writer.WriteStartArray();
var lastEtag = await dumper.ExportAttachments(writer, startEtag, endEtag);
writer.WriteEndArray();
writer.Flush();
// read exported content
var exportedAttachments = RavenJArray.Parse(textStream.GetStringBuilder().ToString());
Assert.Equal(2, exportedAttachments.Count());
Assert.Equal("02000000-0000-0001-0000-000000000006", exportedAttachments.First().Value <string>("Etag"));
Assert.Equal("02000000-0000-0001-0000-000000000007", exportedAttachments.Last().Value <string>("Etag"));
Assert.Equal("02000000-0000-0001-0000-000000000007", lastEtag.ToString());
}
using (var textStream = new StringWriter())
using (var writer = new JsonTextWriter(textStream))
{
var dumper = new CustomDataDumper(store.SystemDatabase);
var startEtag = store.SystemDatabase.Statistics.LastAttachmentEtag.IncrementBy(-5);
writer.WriteStartArray();
var lastEtag = await dumper.ExportAttachments(writer, startEtag, null);
writer.WriteEndArray();
writer.Flush();
// read exported content
var exportedAttachments = RavenJArray.Parse(textStream.GetStringBuilder().ToString());
Assert.Equal(5, exportedAttachments.Count());
Assert.Equal("02000000-0000-0001-0000-000000000006", exportedAttachments.First().Value <string>("Etag"));
Assert.Equal("02000000-0000-0001-0000-00000000000A", exportedAttachments.Last().Value <string>("Etag"));
Assert.Equal("02000000-0000-0001-0000-00000000000A", lastEtag.ToString());
}
WaitForIndexing(store);
store.DatabaseCommands.DeleteByIndex("Raven/DocumentsByEntityName", new IndexQuery()
{
Query = "Tag:Users"
}).WaitForCompletion();
for (var i = 0; i < 10; i++)
{
store.DatabaseCommands.DeleteAttachment("attach/" + (i + 1), null);
}
Etag user6DeletionEtag = null, user9DeletionEtag = null, attach5DeletionEtag = null, attach7DeletionEtag = null;
WaitForUserToContinueTheTest(store);
store.SystemDatabase.TransactionalStorage.Batch(accessor =>
{
user6DeletionEtag =
accessor.Lists.Read(Constants.RavenPeriodicExportsDocsTombstones, "users/6").Etag;
user9DeletionEtag =
accessor.Lists.Read(Constants.RavenPeriodicExportsDocsTombstones, "users/9").Etag;
attach5DeletionEtag =
accessor.Lists.Read(Constants.RavenPeriodicExportsAttachmentsTombstones, "attach/5").Etag;
attach7DeletionEtag =
accessor.Lists.Read(Constants.RavenPeriodicExportsAttachmentsTombstones, "attach/7").Etag;
});
using (var textStream = new StringWriter())
using (var writer = new JsonTextWriter(textStream))
{
var dumper = new CustomDataDumper(store.SystemDatabase);
writer.WriteStartObject();
var lastEtags = new LastEtagsInfo();
var exportResult = new ExportDataResult
{
LastDocDeleteEtag = user6DeletionEtag,
LastAttachmentsDeleteEtag = attach5DeletionEtag
};
lastEtags.LastDocDeleteEtag = user9DeletionEtag;
lastEtags.LastAttachmentsDeleteEtag = attach7DeletionEtag;
dumper.ExportDeletions(writer, exportResult, lastEtags).Wait();
writer.WriteEndObject();
writer.Flush();
// read exported content
var exportJson = RavenJObject.Parse(textStream.GetStringBuilder().ToString());
var docsKeys =
exportJson.Value <RavenJArray>("DocsDeletions").Select(x => x.Value <string>("Key")).ToArray();
var attachmentsKeys =
exportJson.Value <RavenJArray>("AttachmentsDeletions")
.Select(x => x.Value <string>("Key"))
.ToArray();
Assert.Equal(new [] { "users/7", "users/8", "users/9" }, docsKeys);
Assert.Equal(new [] { "attach/6", "attach/7" }, attachmentsKeys);
}
}
}
