public override LastEtagsInfo FetchCurrentMaxEtags()
{
LastEtagsInfo result = null;
database.TransactionalStorage.Batch(accessor =>
{
result = new LastEtagsInfo
{
LastDocsEtag = accessor.Staleness.GetMostRecentDocumentEtag(),
LastAttachmentsEtag = accessor.Staleness.GetMostRecentAttachmentEtag()
};
var lastDocumentTombstone = accessor.Lists.ReadLast(Constants.RavenPeriodicBackupsDocsTombstones);
if (lastDocumentTombstone != null)
result.LastDocDeleteEtag = lastDocumentTombstone.Etag;
var attachmentTombstones =
accessor.Lists.Read(Constants.RavenPeriodicBackupsAttachmentsTombstones, Etag.Empty, null, int.MaxValue)
.OrderBy(x => x.Etag).ToArray();
if (attachmentTombstones.Any())
{
result.LastAttachmentsDeleteEtag = attachmentTombstones.Last().Etag;
}
});
return result;
}
/// <summary>
///
/// </summary>
/// <param name="jsonWriter"></param>
/// <param name="options"></param>
/// <param name="result"></param>
/// <param name="maxEtags">Max etags are inclusive</param>
protected async override void ExportDeletions(JsonTextWriter jsonWriter, SmugglerOptions options, ExportDataResult result, LastEtagsInfo maxEtags)
{
jsonWriter.WritePropertyName("DocsDeletions");
jsonWriter.WriteStartArray();
result.LastDocDeleteEtag = await ExportDocumentsDeletion(options, jsonWriter, result.LastDocDeleteEtag, maxEtags.LastDocDeleteEtag.IncrementBy(1));
jsonWriter.WriteEndArray();
jsonWriter.WritePropertyName("AttachmentsDeletions");
jsonWriter.WriteStartArray();
result.LastAttachmentsDeleteEtag = await ExportAttachmentsDeletion(options, jsonWriter, result.LastAttachmentsDeleteEtag, maxEtags.LastAttachmentsDeleteEtag.IncrementBy(1));
jsonWriter.WriteEndArray();
}
public async Task DataDumperExportHandlesMaxEtagCorrectly()
{
using (var store = NewDocumentStore())
{
using (var session = store.OpenSession())
{
for (var i = 0; i < 10; i++)
{
session.Store(new User { Name = "oren #" + (i+1) });
}
session.SaveChanges();
}
using (var textStream = new StringWriter())
using (var writer = new JsonTextWriter(textStream))
{
var dumper = new CustomDataDumper(store.DocumentDatabase)
{
SmugglerOptions = new SmugglerOptions()
};
var startEtag = store.DocumentDatabase.Statistics.LastDocEtag.IncrementBy(-5);
var endEtag = startEtag.IncrementBy(2);
writer.WriteStartArray();
var lastEtag = await dumper.ExportDocuments(new SmugglerOptions(), writer, startEtag, endEtag);
writer.WriteEndArray();
writer.Flush();
// read exported content
var exportedDocs = RavenJArray.Parse(textStream.GetStringBuilder().ToString());
Assert.Equal(2, exportedDocs.Count());
Assert.Equal("01000000-0000-0001-0000-000000000007", exportedDocs.First().Value<RavenJObject>("@metadata").Value<string>("@etag"));
Assert.Equal("01000000-0000-0001-0000-000000000008", exportedDocs.Last().Value<RavenJObject>("@metadata").Value<string>("@etag"));
Assert.Equal("01000000-0000-0001-0000-000000000008", lastEtag.ToString());
}
using (var textStream = new StringWriter())
using (var writer = new JsonTextWriter(textStream))
{
var dumper = new CustomDataDumper(store.DocumentDatabase)
{
SmugglerOptions = new SmugglerOptions()
};
var startEtag = store.DocumentDatabase.Statistics.LastDocEtag.IncrementBy(-5);
writer.WriteStartArray();
var lastEtag = await dumper.ExportDocuments(new SmugglerOptions(), writer, startEtag, null);
writer.WriteEndArray();
writer.Flush();
// read exported content
var exportedDocs = RavenJArray.Parse(textStream.GetStringBuilder().ToString());
Assert.Equal(5, exportedDocs.Count());
Assert.Equal("01000000-0000-0001-0000-000000000007", exportedDocs.First().Value<RavenJObject>("@metadata").Value<string>("@etag"));
Assert.Equal("01000000-0000-0001-0000-00000000000B", exportedDocs.Last().Value<RavenJObject>("@metadata").Value<string>("@etag"));
Assert.Equal("01000000-0000-0001-0000-00000000000B", lastEtag.ToString());
}
for (var i = 0; i < 10; i++)
{
store.DatabaseCommands.PutAttachment("attach/" + (i+1), null, new MemoryStream(new [] { (byte)i }), new RavenJObject());
}
using (var textStream = new StringWriter())
using (var writer = new JsonTextWriter(textStream))
{
var dumper = new CustomDataDumper(store.DocumentDatabase)
{
SmugglerOptions = new SmugglerOptions()
};
var startEtag = store.DocumentDatabase.Statistics.LastAttachmentEtag.IncrementBy(-5);
var endEtag = startEtag.IncrementBy(2);
writer.WriteStartArray();
var lastEtag = await dumper.ExportAttachments(writer, startEtag, endEtag);
writer.WriteEndArray();
writer.Flush();
// read exported content
var exportedAttachments = RavenJArray.Parse(textStream.GetStringBuilder().ToString());
Assert.Equal(2, exportedAttachments.Count());
Assert.Equal("02000000-0000-0001-0000-000000000006", exportedAttachments.First().Value<string>("Etag"));
Assert.Equal("02000000-0000-0001-0000-000000000007", exportedAttachments.Last().Value<string>("Etag"));
Assert.Equal("02000000-0000-0001-0000-000000000007", lastEtag.ToString());
}
using (var textStream = new StringWriter())
using (var writer = new JsonTextWriter(textStream))
{
var dumper = new CustomDataDumper(store.DocumentDatabase)
{
SmugglerOptions = new SmugglerOptions()
};
var startEtag = store.DocumentDatabase.Statistics.LastAttachmentEtag.IncrementBy(-5);
writer.WriteStartArray();
var lastEtag = await dumper.ExportAttachments(writer, startEtag, null);
writer.WriteEndArray();
writer.Flush();
// read exported content
var exportedAttachments = RavenJArray.Parse(textStream.GetStringBuilder().ToString());
Assert.Equal(5, exportedAttachments.Count());
Assert.Equal("02000000-0000-0001-0000-000000000006", exportedAttachments.First().Value<string>("Etag"));
Assert.Equal("02000000-0000-0001-0000-00000000000A", exportedAttachments.Last().Value<string>("Etag"));
Assert.Equal("02000000-0000-0001-0000-00000000000A", lastEtag.ToString());
}
WaitForIndexing(store);
store.DatabaseCommands.DeleteByIndex("Raven/DocumentsByEntityName", new IndexQuery()
{
Query = "Tag:Users"
}).WaitForCompletion();
for (var i = 0; i < 10; i++)
{
store.DatabaseCommands.DeleteAttachment("attach/" + (i+1), null);
}
Etag user6DeletionEtag = null, user9DeletionEtag = null, attach5DeletionEtag = null, attach7DeletionEtag = null;
WaitForUserToContinueTheTest(store);
store.DocumentDatabase.TransactionalStorage.Batch(accessor =>
{
user6DeletionEtag =
accessor.Lists.Read(Constants.RavenPeriodicExportsDocsTombstones, "users/6").Etag;
user9DeletionEtag =
accessor.Lists.Read(Constants.RavenPeriodicExportsDocsTombstones, "users/9").Etag;
attach5DeletionEtag =
accessor.Lists.Read(Constants.RavenPeriodicExportsAttachmentsTombstones, "attach/5").Etag;
attach7DeletionEtag =
accessor.Lists.Read(Constants.RavenPeriodicExportsAttachmentsTombstones, "attach/7").Etag;
});
using (var textStream = new StringWriter())
using (var writer = new JsonTextWriter(textStream))
{
var dumper = new CustomDataDumper(store.DocumentDatabase)
{
SmugglerOptions = new SmugglerOptions()
};
writer.WriteStartObject();
var lastEtags = new LastEtagsInfo();
var exportResult = new ExportDataResult
{
LastDocDeleteEtag = user6DeletionEtag,
LastAttachmentsDeleteEtag = attach5DeletionEtag
};
lastEtags.LastDocDeleteEtag = user9DeletionEtag;
lastEtags.LastAttachmentsDeleteEtag = attach7DeletionEtag;
dumper.ExportDeletions(writer, new SmugglerOptions(), exportResult, lastEtags);
writer.WriteEndObject();
writer.Flush();
// read exported content
var exportJson = RavenJObject.Parse(textStream.GetStringBuilder().ToString());
var docsKeys =
exportJson.Value<RavenJArray>("DocsDeletions").Select(x => x.Value<string>("Key")).ToArray();
var attachmentsKeys =
exportJson.Value<RavenJArray>("AttachmentsDeletions")
.Select(x => x.Value<string>("Key"))
.ToArray();
Assert.Equal(new [] { "users/7", "users/8", "users/9" }, docsKeys);
Assert.Equal(new [] { "attach/6", "attach/7" }, attachmentsKeys);
}
}
}
public new void ExportDeletions(JsonTextWriter jsonWriter, SmugglerOptions options, ExportDataResult result, LastEtagsInfo maxEtags)
{
base.ExportDeletions(jsonWriter, options, result, maxEtags);
}
protected override void ExportDeletions(JsonTextWriter jsonWriter, SmugglerOptions options, ExportDataResult result,
LastEtagsInfo maxEtagsToFetch)
{
throw new NotImplementedException("Exporting deletions is not supported for Command Line Smuggler");
}
protected abstract void ExportDeletions(JsonTextWriter jsonWriter, SmugglerOptions options, ExportDataResult result, LastEtagsInfo maxEtagsToFetch);
protected abstract void ExportDeletions(JsonTextWriter jsonWriter, SmugglerOptions options, ExportDataResult result, LastEtagsInfo maxEtagsToFetch);
public abstract Task ExportDeletions(JsonTextWriter jsonWriter, OperationState result, LastEtagsInfo maxEtagsToFetch);
public abstract Task ExportDeletions(JsonTextWriter jsonWriter, ExportDataResult result, LastEtagsInfo maxEtagsToFetch);