public override void ExecuteResult(ControllerContext context)
{
if (context == null)
{
throw new ArgumentNullException("context");
}
HttpResponseBase response = context.HttpContext.Response;
// Incredibly, IE *still* doesn't know what JSON is, so we have to trick it to avoid prompting the user to save the returning JSON
if (context.RequestContext.HttpContext.Request["ie"] != null && Convert.ToBoolean(context.RequestContext.HttpContext.Request["ie"]) == true)
{
response.ContentType = "text/html";
}
else
{
response.ContentType = "application/json";
}
if (Data != null)
{
// HACK: RavenDB uses an internalised version of JSON.Net 4.0.8, while Bowerbird references JSON.Net 4.5. When RavenDB returns a DESERIALISED v4.0.8 object
// and then I try to SERIALISE it with v4.5, the output is rubbish. For some reason JArray types don't get serialised at all.
#if JS_COMBINE_MINIFY
var formatting = Raven.Imports.Newtonsoft.Json.Formatting.None;
#else
var formatting = Raven.Imports.Newtonsoft.Json.Formatting.Indented;
#endif
var writer = new Raven.Imports.Newtonsoft.Json.JsonTextWriter(response.Output)
{
Formatting = formatting
};
var serializer = new Raven.Imports.Newtonsoft.Json.JsonSerializer();
serializer.Serialize(writer, Data);
writer.Flush();
//var writer = new Newtonsoft.Json.JsonTextWriter(response.Output) { Formatting = Newtonsoft.Json.Formatting.Indented };
//var serializer = new Newtonsoft.Json.JsonSerializer();
//serializer.Serialize(writer, Data);
//writer.Flush();
}
}
public override void ExecuteResult(ControllerContext context)
{
if (context == null)
{
throw new ArgumentNullException("context");
}
HttpResponseBase response = context.HttpContext.Response;
// Incredibly, IE *still* doesn't know what JSON is, so we have to trick it to avoid prompting the user to save the returning JSON
if (context.RequestContext.HttpContext.Request["ie"] != null && Convert.ToBoolean(context.RequestContext.HttpContext.Request["ie"]) == true)
{
response.ContentType = "text/html";
}
else
{
response.ContentType = "application/json";
}
if (Data != null)
{
// HACK: RavenDB uses an internalised version of JSON.Net 4.0.8, while Bowerbird references JSON.Net 4.5. When RavenDB returns a DESERIALISED v4.0.8 object
// and then I try to SERIALISE it with v4.5, the output is rubbish. For some reason JArray types don't get serialised at all.
#if JS_COMBINE_MINIFY
var formatting = Raven.Imports.Newtonsoft.Json.Formatting.None;
#else
var formatting = Raven.Imports.Newtonsoft.Json.Formatting.Indented;
#endif
var writer = new Raven.Imports.Newtonsoft.Json.JsonTextWriter(response.Output) { Formatting = formatting };
var serializer = new Raven.Imports.Newtonsoft.Json.JsonSerializer();
serializer.Serialize(writer, Data);
writer.Flush();
//var writer = new Newtonsoft.Json.JsonTextWriter(response.Output) { Formatting = Newtonsoft.Json.Formatting.Indented };
//var serializer = new Newtonsoft.Json.JsonSerializer();
//serializer.Serialize(writer, Data);
//writer.Flush();
}
}
public Task<Etag> ExportAttachments(JsonTextWriter jsonWriter, Etag lastEtag, Etag maxEtag)
{
Operations.Initialize(Options);
return ExportAttachments(new RavenConnectionStringOptions(), jsonWriter, lastEtag, maxEtag);
}
public new void ExportDeletions(JsonTextWriter jsonWriter, SmugglerOptions options, ExportDataResult result, LastEtagsInfo maxEtags)
{
base.ExportDeletions(jsonWriter, options, result, maxEtags);
}
public new Task<Etag> ExportAttachments(JsonTextWriter jsonWriter, Etag lastEtag, Etag maxEtag)
{
return base.ExportAttachments(new RavenConnectionStringOptions(), jsonWriter, lastEtag, maxEtag);
}
public new Task<Etag> ExportDocuments(SmugglerOptions options, JsonTextWriter jsonWriter, Etag lastEtag, Etag maxEtag)
{
return base.ExportDocuments(new RavenConnectionStringOptions(), options, jsonWriter, lastEtag, maxEtag);
}
public Task<ExportOperationStatus> ExportAttachments(JsonTextWriter jsonWriter, Etag lastEtag, Etag maxEtag, int maxNumberOfAttachmentsToExport)
{
Operations.Initialize(Options);
return ExportAttachments(new RavenConnectionStringOptions(), jsonWriter, lastEtag, maxEtag, maxNumberOfAttachmentsToExport);
}
public Task <Etag> ExportDocuments(SmugglerOptions options, JsonTextWriter jsonWriter, Etag lastEtag, Etag maxEtag)
{
return(base.ExportDocuments(new RavenConnectionStringOptions(), options, jsonWriter, lastEtag, maxEtag));
}
public override Task ExportDeletions(JsonTextWriter jsonWriter, ExportDataResult result, LastEtagsInfo maxEtags)
{
return(base.ExportDeletions(jsonWriter, result, maxEtags));
}
public async Task DataDumperExportHandlesMaxEtagCorrectly()
{
using (var store = NewDocumentStore())
{
using (var session = store.OpenSession())
{
for (var i = 0; i < 10; i++)
{
session.Store(new User {
Name = "oren #" + (i + 1)
});
}
session.SaveChanges();
}
using (var textStream = new StringWriter())
using (var writer = new JsonTextWriter(textStream))
{
var dumper = new CustomDataDumper(store.SystemDatabase);
var startEtag = store.SystemDatabase.Statistics.LastDocEtag.IncrementBy(-5);
var endEtag = startEtag.IncrementBy(2);
writer.WriteStartArray();
var lastEtag = await dumper.ExportDocuments(writer, startEtag, endEtag);
writer.WriteEndArray();
writer.Flush();
// read exported content
var exportedDocs = RavenJArray.Parse(textStream.GetStringBuilder().ToString());
Assert.Equal(2, exportedDocs.Count());
Assert.Equal("01000000-0000-0001-0000-000000000007", exportedDocs.First().Value <RavenJObject>("@metadata").Value <string>("@etag"));
Assert.Equal("01000000-0000-0001-0000-000000000008", exportedDocs.Last().Value <RavenJObject>("@metadata").Value <string>("@etag"));
Assert.Equal("01000000-0000-0001-0000-000000000008", lastEtag.ToString());
}
using (var textStream = new StringWriter())
using (var writer = new JsonTextWriter(textStream))
{
var dumper = new CustomDataDumper(store.SystemDatabase);
var startEtag = store.SystemDatabase.Statistics.LastDocEtag.IncrementBy(-5);
writer.WriteStartArray();
var lastEtag = await dumper.ExportDocuments(writer, startEtag, null);
writer.WriteEndArray();
writer.Flush();
// read exported content
var exportedDocs = RavenJArray.Parse(textStream.GetStringBuilder().ToString());
Assert.Equal(5, exportedDocs.Count());
Assert.Equal("01000000-0000-0001-0000-000000000007", exportedDocs.First().Value <RavenJObject>("@metadata").Value <string>("@etag"));
Assert.Equal("01000000-0000-0001-0000-00000000000B", exportedDocs.Last().Value <RavenJObject>("@metadata").Value <string>("@etag"));
Assert.Equal("01000000-0000-0001-0000-00000000000B", lastEtag.ToString());
}
for (var i = 0; i < 10; i++)
{
store.DatabaseCommands.PutAttachment("attach/" + (i + 1), null, new MemoryStream(new [] { (byte)i }), new RavenJObject());
}
using (var textStream = new StringWriter())
using (var writer = new JsonTextWriter(textStream))
{
var dumper = new CustomDataDumper(store.SystemDatabase);
var startEtag = store.SystemDatabase.Statistics.LastAttachmentEtag.IncrementBy(-5);
var endEtag = startEtag.IncrementBy(2);
writer.WriteStartArray();
var lastEtag = await dumper.ExportAttachments(writer, startEtag, endEtag);
writer.WriteEndArray();
writer.Flush();
// read exported content
var exportedAttachments = RavenJArray.Parse(textStream.GetStringBuilder().ToString());
Assert.Equal(2, exportedAttachments.Count());
Assert.Equal("02000000-0000-0001-0000-000000000006", exportedAttachments.First().Value <string>("Etag"));
Assert.Equal("02000000-0000-0001-0000-000000000007", exportedAttachments.Last().Value <string>("Etag"));
Assert.Equal("02000000-0000-0001-0000-000000000007", lastEtag.ToString());
}
using (var textStream = new StringWriter())
using (var writer = new JsonTextWriter(textStream))
{
var dumper = new CustomDataDumper(store.SystemDatabase);
var startEtag = store.SystemDatabase.Statistics.LastAttachmentEtag.IncrementBy(-5);
writer.WriteStartArray();
var lastEtag = await dumper.ExportAttachments(writer, startEtag, null);
writer.WriteEndArray();
writer.Flush();
// read exported content
var exportedAttachments = RavenJArray.Parse(textStream.GetStringBuilder().ToString());
Assert.Equal(5, exportedAttachments.Count());
Assert.Equal("02000000-0000-0001-0000-000000000006", exportedAttachments.First().Value <string>("Etag"));
Assert.Equal("02000000-0000-0001-0000-00000000000A", exportedAttachments.Last().Value <string>("Etag"));
Assert.Equal("02000000-0000-0001-0000-00000000000A", lastEtag.ToString());
}
WaitForIndexing(store);
store.DatabaseCommands.DeleteByIndex("Raven/DocumentsByEntityName", new IndexQuery()
{
Query = "Tag:Users"
}).WaitForCompletion();
for (var i = 0; i < 10; i++)
{
store.DatabaseCommands.DeleteAttachment("attach/" + (i + 1), null);
}
Etag user6DeletionEtag = null, user9DeletionEtag = null, attach5DeletionEtag = null, attach7DeletionEtag = null;
WaitForUserToContinueTheTest(store);
store.SystemDatabase.TransactionalStorage.Batch(accessor =>
{
user6DeletionEtag =
accessor.Lists.Read(Constants.RavenPeriodicExportsDocsTombstones, "users/6").Etag;
user9DeletionEtag =
accessor.Lists.Read(Constants.RavenPeriodicExportsDocsTombstones, "users/9").Etag;
attach5DeletionEtag =
accessor.Lists.Read(Constants.RavenPeriodicExportsAttachmentsTombstones, "attach/5").Etag;
attach7DeletionEtag =
accessor.Lists.Read(Constants.RavenPeriodicExportsAttachmentsTombstones, "attach/7").Etag;
});
using (var textStream = new StringWriter())
using (var writer = new JsonTextWriter(textStream))
{
var dumper = new CustomDataDumper(store.SystemDatabase);
writer.WriteStartObject();
var lastEtags = new LastEtagsInfo();
var exportResult = new ExportDataResult
{
LastDocDeleteEtag = user6DeletionEtag,
LastAttachmentsDeleteEtag = attach5DeletionEtag
};
lastEtags.LastDocDeleteEtag = user9DeletionEtag;
lastEtags.LastAttachmentsDeleteEtag = attach7DeletionEtag;
dumper.ExportDeletions(writer, exportResult, lastEtags).Wait();
writer.WriteEndObject();
writer.Flush();
// read exported content
var exportJson = RavenJObject.Parse(textStream.GetStringBuilder().ToString());
var docsKeys =
exportJson.Value <RavenJArray>("DocsDeletions").Select(x => x.Value <string>("Key")).ToArray();
var attachmentsKeys =
exportJson.Value <RavenJArray>("AttachmentsDeletions")
.Select(x => x.Value <string>("Key"))
.ToArray();
Assert.Equal(new [] { "users/7", "users/8", "users/9" }, docsKeys);
Assert.Equal(new [] { "attach/6", "attach/7" }, attachmentsKeys);
}
}
}
public Task <Etag> ExportAttachments(JsonTextWriter jsonWriter, Etag lastEtag, Etag maxEtag)
{
Operations.Initialize(SmugglerOptions);
return(ExportAttachments(new RavenConnectionStringOptions(), jsonWriter, lastEtag, maxEtag));
}
public new void ExportDeletions(JsonTextWriter jsonWriter, SmugglerOptions options, ExportDataResult result, LastEtagsInfo maxEtags)
{
base.ExportDeletions(jsonWriter, options, result, maxEtags);
}
public Task <Etag> ExportAttachments(JsonTextWriter jsonWriter, Etag lastEtag, Etag maxEtag)
{
return(base.ExportAttachments(new RavenConnectionStringOptions(), jsonWriter, lastEtag, maxEtag));
}
public override Task ExportDeletions(JsonTextWriter jsonWriter, OperationState result, LastEtagsInfo maxEtags)
{
return base.ExportDeletions(jsonWriter, result, maxEtags);
}
public Task ExportDeletions(JsonTextWriter jsonWriter, OperationState result, Etag lastDocDeleteEtag, Etag lastAttachmentsDeleteEtag)
{
var maxEtags = new LastEtagsInfo
{
LastDocDeleteEtag = lastDocDeleteEtag,
LastAttachmentsDeleteEtag = lastAttachmentsDeleteEtag
};
return base.ExportDeletions(jsonWriter, result, maxEtags);
}
public async Task DataDumperExportHandlesMaxEtagCorrectly()
{
using (var store = NewDocumentStore())
{
using (var session = store.OpenSession())
{
for (var i = 0; i < 10; i++)
{
session.Store(new User { Name = "oren #" + (i + 1) });
}
session.SaveChanges();
}
using (var textStream = new StringWriter())
using (var writer = new JsonTextWriter(textStream))
{
var dumper = new CustomDataDumper(store.SystemDatabase);
var startEtag = store.SystemDatabase.Statistics.LastDocEtag.IncrementBy(-5);
var endEtag = startEtag.IncrementBy(2);
writer.WriteStartArray();
var lastEtag = await dumper.ExportDocuments(writer, startEtag, endEtag);
writer.WriteEndArray();
writer.Flush();
// read exported content
var exportedDocs = RavenJArray.Parse(textStream.GetStringBuilder().ToString());
Assert.Equal(2, exportedDocs.Count());
Assert.Equal("01000000-0000-0001-0000-000000000007", exportedDocs.First().Value<RavenJObject>("@metadata").Value<string>("@etag"));
Assert.Equal("01000000-0000-0001-0000-000000000008", exportedDocs.Last().Value<RavenJObject>("@metadata").Value<string>("@etag"));
Assert.Equal("01000000-0000-0001-0000-000000000008", lastEtag.ToString());
}
using (var textStream = new StringWriter())
using (var writer = new JsonTextWriter(textStream))
{
var dumper = new CustomDataDumper(store.SystemDatabase);
var startEtag = store.SystemDatabase.Statistics.LastDocEtag.IncrementBy(-5);
writer.WriteStartArray();
var lastEtag = await dumper.ExportDocuments(writer, startEtag, null);
writer.WriteEndArray();
writer.Flush();
// read exported content
var exportedDocs = RavenJArray.Parse(textStream.GetStringBuilder().ToString());
Assert.Equal(5, exportedDocs.Count());
Assert.Equal("01000000-0000-0001-0000-000000000007", exportedDocs.First().Value<RavenJObject>("@metadata").Value<string>("@etag"));
Assert.Equal("01000000-0000-0001-0000-00000000000B", exportedDocs.Last().Value<RavenJObject>("@metadata").Value<string>("@etag"));
Assert.Equal("01000000-0000-0001-0000-00000000000B", lastEtag.ToString());
}
for (var i = 0; i < 10; i++)
{
store.DatabaseCommands.PutAttachment("attach/" + (i + 1), null, new MemoryStream(new[] { (byte)i }), new RavenJObject());
}
using (var textStream = new StringWriter())
using (var writer = new JsonTextWriter(textStream))
{
var dumper = new CustomDataDumper(store.SystemDatabase);
var startEtag = store.SystemDatabase.Statistics.LastAttachmentEtag.IncrementBy(-5);
var endEtag = startEtag.IncrementBy(2);
writer.WriteStartArray();
var lastEtag = await dumper.ExportAttachments(writer, startEtag, endEtag);
writer.WriteEndArray();
writer.Flush();
// read exported content
var exportedAttachments = RavenJArray.Parse(textStream.GetStringBuilder().ToString());
Assert.Equal(2, exportedAttachments.Count());
Assert.Equal("02000000-0000-0001-0000-000000000006", exportedAttachments.First().Value<string>("Etag"));
Assert.Equal("02000000-0000-0001-0000-000000000007", exportedAttachments.Last().Value<string>("Etag"));
Assert.Equal("02000000-0000-0001-0000-000000000007", lastEtag.ToString());
}
using (var textStream = new StringWriter())
using (var writer = new JsonTextWriter(textStream))
{
var dumper = new CustomDataDumper(store.SystemDatabase);
var startEtag = store.SystemDatabase.Statistics.LastAttachmentEtag.IncrementBy(-5);
writer.WriteStartArray();
var lastEtag = await dumper.ExportAttachments(writer, startEtag, null);
writer.WriteEndArray();
writer.Flush();
// read exported content
var exportedAttachments = RavenJArray.Parse(textStream.GetStringBuilder().ToString());
Assert.Equal(5, exportedAttachments.Count());
Assert.Equal("02000000-0000-0001-0000-000000000006", exportedAttachments.First().Value<string>("Etag"));
Assert.Equal("02000000-0000-0001-0000-00000000000A", exportedAttachments.Last().Value<string>("Etag"));
Assert.Equal("02000000-0000-0001-0000-00000000000A", lastEtag.ToString());
}
WaitForIndexing(store);
store.DatabaseCommands.DeleteByIndex("Raven/DocumentsByEntityName", new IndexQuery()
{
Query = "Tag:Users"
}).WaitForCompletion();
for (var i = 0; i < 10; i++)
{
store.DatabaseCommands.DeleteAttachment("attach/" + (i + 1), null);
}
Etag user6DeletionEtag = null, user9DeletionEtag = null, attach5DeletionEtag = null, attach7DeletionEtag = null;
WaitForUserToContinueTheTest(store);
store.SystemDatabase.TransactionalStorage.Batch(accessor =>
{
user6DeletionEtag =
accessor.Lists.Read(Constants.RavenPeriodicExportsDocsTombstones, "users/6").Etag;
user9DeletionEtag =
accessor.Lists.Read(Constants.RavenPeriodicExportsDocsTombstones, "users/9").Etag;
attach5DeletionEtag =
accessor.Lists.Read(Constants.RavenPeriodicExportsAttachmentsTombstones, "attach/5").Etag;
attach7DeletionEtag =
accessor.Lists.Read(Constants.RavenPeriodicExportsAttachmentsTombstones, "attach/7").Etag;
});
using (var textStream = new StringWriter())
using (var writer = new JsonTextWriter(textStream))
{
var dumper = new CustomDataDumper(store.SystemDatabase);
writer.WriteStartObject();
var lastEtags = new LastEtagsInfo();
var exportResult = new OperationState
{
LastDocDeleteEtag = user6DeletionEtag,
LastAttachmentsDeleteEtag = attach5DeletionEtag
};
lastEtags.LastDocDeleteEtag = user9DeletionEtag;
lastEtags.LastAttachmentsDeleteEtag = attach7DeletionEtag;
dumper.ExportDeletions(writer, exportResult, lastEtags).Wait();
writer.WriteEndObject();
writer.Flush();
// read exported content
var exportJson = RavenJObject.Parse(textStream.GetStringBuilder().ToString());
var docsKeys =
exportJson.Value<RavenJArray>("DocsDeletions").Select(x => x.Value<string>("Key")).ToArray();
var attachmentsKeys =
exportJson.Value<RavenJArray>("AttachmentsDeletions")
.Select(x => x.Value<string>("Key"))
.ToArray();
Assert.Equal(new[] { "users/7", "users/8", "users/9" }, docsKeys);
Assert.Equal(new[] { "attach/6", "attach/7" }, attachmentsKeys);
}
}
}
public static void WaitForIndexing(IDocumentStore store, string database = null, TimeSpan?timeout = null)
{
var databaseCommands = store.DatabaseCommands;
if (database != null)
{
databaseCommands = databaseCommands.ForDatabase(database);
}
timeout = timeout ?? (Debugger.IsAttached
? TimeSpan.FromMinutes(15)
: TimeSpan.FromMinutes(1));
var sp = Stopwatch.StartNew();
while (sp.Elapsed < timeout.Value)
{
var databaseStatistics = databaseCommands.GetStatistics();
if (databaseStatistics.Indexes.All(x => x.IsStale == false))
{
return;
}
if (databaseStatistics.Indexes.Any(x => x.State == IndexState.Error))
{
break;
}
Thread.Sleep(32);
}
var request = databaseCommands.CreateRequest("/indexes/performance", HttpMethod.Get);
var perf = request.ReadResponseJson();
request = databaseCommands.CreateRequest("/indexes/errors", HttpMethod.Get);
var errors = request.ReadResponseJson();
var total = new RavenJObject
{
["Errors"] = errors,
["Performance"] = perf
};
var file = Path.GetTempFileName() + ".json";
using (var writer = File.CreateText(file))
{
var jsonTextWriter = new JsonTextWriter(writer);
total.WriteTo(jsonTextWriter);
jsonTextWriter.Flush();
}
var stats = databaseCommands.GetStatistics();
var corrupted = stats.Indexes.Where(x => x.State == IndexState.Error).ToList();
if (corrupted.Count > 0)
{
throw new InvalidOperationException(
$"The following indexes are with error state: {string.Join(",", corrupted.Select(x => x.Name))} - details at " + file);
}
throw new TimeoutException("The indexes stayed stale for more than " + timeout.Value + ", stats at " + file);
}
