protected override Task <IAsyncEnumerator <RavenJObject> > GetDocuments(Etag lastEtag)
{
if (IsDocsStreamingSupported)
{
ShowProgress("Streaming documents from " + lastEtag);
return(Commands.StreamDocsAsync(lastEtag));
}
int retries = RetriesCount;
while (true)
{
try
{
RavenJArray documents = null;
var url = "/docs?pageSize=" + SmugglerOptions.BatchSize + "&etag=" + lastEtag;
ShowProgress("GET " + url);
var request = CreateRequest(url);
request.ExecuteRequest(reader => documents = RavenJArray.Load(new JsonTextReader(reader)));
return(new CompletedTask <IAsyncEnumerator <RavenJObject> >(new AsyncEnumeratorBridge <RavenJObject>(documents.Values <RavenJObject>().GetEnumerator())));
}
catch (Exception e)
{
if (retries-- == 0)
{
throw;
}
LastRequestErrored = true;
ShowProgress("Error reading from database, remaining attempts {0}, will retry. Error: {1}", retries, e, RetriesCount);
}
}
}
public async Task <RavenJArray> ReadJsonArrayAsync()
{
using (var stream = await InnerRequest.Content.ReadAsStreamAsync())
using (var streamReader = new StreamReader(stream, GetRequestEncoding()))
using (var jsonReader = new RavenJsonTextReader(streamReader))
return(RavenJArray.Load(jsonReader));
}
protected async override Task <IAsyncEnumerator <RavenJObject> > GetDocuments(RavenConnectionStringOptions src, Etag lastEtag, int take)
{
if (IsDocsStreamingSupported)
{
ShowProgress("Streaming documents from {0}, batch size {1}", lastEtag, take);
return(await Commands.StreamDocsAsync(lastEtag, pageSize : take));
}
int retries = RetriesCount;
while (true)
{
try
{
RavenJArray documents = null;
var url = "/docs?pageSize=" + Math.Min(SmugglerOptions.BatchSize, take) + "&etag=" + lastEtag;
ShowProgress("GET " + url);
var request = CreateRequest(src, url);
request.ExecuteRequest(reader => documents = RavenJArray.Load(new JsonTextReader(reader)));
return(new AsyncEnumeratorBridge <RavenJObject>(documents.Values <RavenJObject>().GetEnumerator()));
}
catch (Exception e)
{
if (retries-- == 0)
{
throw;
}
LastRequestErrored = true;
ShowProgress("Error reading from database, remaining attempts {0}, will retry. Error: {1}", retries, e);
}
}
}
private void ExportDocuments(SmugglerOptions options, JsonTextWriter jsonWriter)
{
var lastEtag = Guid.Empty;
int totalCount = 0;
while (true)
{
RavenJArray documents = null;
var request = CreateRequest("/docs?pageSize=128&etag=" + lastEtag);
request.ExecuteRequest(reader => documents = RavenJArray.Load(new JsonTextReader(reader)));
if (documents.Length == 0)
{
Console.WriteLine("Done with reading documents, total: {0}", totalCount);
break;
}
var final = documents.Where(options.MatchFilters).ToList();
final.ForEach(item => item.WriteTo(jsonWriter));
totalCount += final.Count;
Console.WriteLine("Reading batch of {0,3} documents, read so far: {1,10:#,#;;0}", documents.Length, totalCount);
lastEtag = new Guid(documents.Last().Value <RavenJObject>("@metadata").Value <string>("@etag"));
}
}
protected override RavenJArray GetDocuments(Guid lastEtag)
{
RavenJArray documents = null;
var request = CreateRequest("/docs?pageSize=128&etag=" + lastEtag);
request.ExecuteRequest(reader => documents = RavenJArray.Load(new JsonTextReader(reader)));
return(documents);
}
protected override RavenJArray GetIndexes(int totalCount)
{
RavenJArray indexes = null;
var request = CreateRequest("/indexes?pageSize=128&start=" + totalCount);
request.ExecuteRequest(reader => indexes = RavenJArray.Load(new JsonTextReader(reader)));
return(indexes);
}
protected override Task <RavenJArray> GetIndexes(int totalCount)
{
RavenJArray indexes = null;
var request = CreateRequest("/indexes?pageSize=" + SmugglerOptions.BatchSize + "&start=" + totalCount);
request.ExecuteRequest(reader => indexes = RavenJArray.Load(new JsonTextReader(reader)));
return(new CompletedTask <RavenJArray>(indexes));
}
protected async override Task <RavenJArray> GetIndexes(RavenConnectionStringOptions src, int totalCount)
{
RavenJArray indexes = null;
var request = CreateRequest(src, "/indexes?pageSize=" + SmugglerOptions.BatchSize + "&start=" + totalCount);
request.ExecuteRequest(reader => indexes = RavenJArray.Load(new JsonTextReader(reader)));
return(indexes);
}
protected async Task <RavenJArray> ReadJsonArrayAsync()
{
using (var stream = await InnerRequest.Content.ReadAsStreamAsync().ConfigureAwait(false))
using (var buffered = new BufferedStream(stream))
using (var streamReader = new StreamReader(buffered, GetRequestEncoding()))
using (var jsonReader = new RavenJsonTextReader(streamReader))
{
return(RavenJArray.Load(jsonReader));
}
}
protected override Task <RavenJArray> GetTransformers(int start)
{
if (IsTransformersSupported == false)
{
return(new CompletedTask <RavenJArray>(new RavenJArray()));
}
RavenJArray transformers = null;
var request = CreateRequest("/transformers?pageSize=" + SmugglerOptions.BatchSize + "&start=" + start);
request.ExecuteRequest(reader => transformers = RavenJArray.Load(new JsonTextReader(reader)));
return(new CompletedTask <RavenJArray>(transformers));
}
protected async override Task <RavenJArray> GetTransformers(RavenConnectionStringOptions src, int start)
{
if (IsTransformersSupported == false)
{
return(new RavenJArray());
}
RavenJArray transformers = null;
var request = CreateRequest(src, "/transformers?pageSize=" + SmugglerOptions.BatchSize + "&start=" + start);
request.ExecuteRequest(reader => transformers = RavenJArray.Load(new JsonTextReader(reader)));
return(transformers);
}
protected override Task <RavenJArray> GetTransformers(int totalCount)
{
if (Mode == SmugglerMode.Legacy)
{
return(new CompletedTask <RavenJArray>(new RavenJArray()));
}
RavenJArray transformers = null;
var request = CreateRequest("/transformers?pageSize=" + SmugglerOptions.BatchSize + "&start=" + totalCount);
request.ExecuteRequest(reader => transformers = RavenJArray.Load(new JsonTextReader(reader)));
return(new CompletedTask <RavenJArray>(transformers));
}
///<summary>
/// Get the possible terms for the specified field in the index asynchronously
/// You can page through the results by use fromValue parameter as the
/// starting point for the next query
///</summary>
///<returns></returns>
public Task <string[]> GetTermsAsync(string index, string field, string fromValue, int pageSize)
{
return(url.Terms(index, field, fromValue, pageSize)
.NoCache()
.ToJsonRequest(this, credentials, convention)
.ReadResponseStringAsync()
.ContinueWith(task =>
{
using (var reader = new JsonTextReader(new StringReader(task.Result)))
{
var json = RavenJArray.Load(reader);
return json.Select(x => x.Value <string>()).ToArray();
}
}));
}
protected override async Task <Etag> ExportAttachments(JsonTextWriter jsonWriter, Etag lastEtag)
{
int totalCount = 0;
while (true)
{
RavenJArray attachmentInfo = null;
var request = CreateRequest("/static/?pageSize=" + SmugglerOptions.BatchSize + "&etag=" + lastEtag);
request.ExecuteRequest(reader => attachmentInfo = RavenJArray.Load(new JsonTextReader(reader)));
if (attachmentInfo.Length == 0)
{
var databaseStatistics = await GetStats();
var lastEtagComparable = new ComparableByteArray(lastEtag);
if (lastEtagComparable.CompareTo(databaseStatistics.LastAttachmentEtag) < 0)
{
lastEtag = EtagUtil.Increment(lastEtag, SmugglerOptions.BatchSize);
ShowProgress("Got no results but didn't get to the last attachment etag, trying from: {0}", lastEtag);
continue;
}
ShowProgress("Done with reading attachments, total: {0}", totalCount);
return(lastEtag);
}
totalCount += attachmentInfo.Length;
ShowProgress("Reading batch of {0,3} attachments, read so far: {1,10:#,#;;0}", attachmentInfo.Length, totalCount);
foreach (var item in attachmentInfo)
{
ShowProgress("Downloading attachment: {0}", item.Value <string>("Key"));
byte[] attachmentData = null;
var requestData = CreateRequest("/static/" + item.Value <string>("Key"));
requestData.ExecuteRequest(reader => attachmentData = reader.ReadData());
new RavenJObject
{
{ "Data", attachmentData },
{ "Metadata", item.Value <RavenJObject>("Metadata") },
{ "Key", item.Value <string>("Key") }
}
.WriteTo(jsonWriter);
}
lastEtag = Etag.Parse(attachmentInfo.Last().Value <string>("Etag"));
}
}
private void ExportAttachments(JsonTextWriter jsonWriter)
{
var lastEtag = Guid.Empty;
int totalCount = 0;
while (true)
{
RavenJArray attachmentInfo = null;
var request = CreateRequest("/static/?pageSize=128&etag=" + lastEtag);
request.ExecuteRequest(reader => attachmentInfo = RavenJArray.Load(new JsonTextReader(reader)));
if (attachmentInfo.Length == 0)
{
Console.WriteLine("Done with reading attachments, total: {0}", totalCount);
break;
}
totalCount += attachmentInfo.Length;
Console.WriteLine("Reading batch of {0,3} attachments, read so far: {1,10:#,#;;0}", attachmentInfo.Length, totalCount);
foreach (var item in attachmentInfo)
{
Console.WriteLine("Downloading attachment: {0}", item.Value <string>("Key"));
byte[] attachmentData = null;
var requestData = CreateRequest("/static/" + item.Value <string>("Key"));
requestData.ExecuteRequest(reader => attachmentData = reader.ReadData());
new RavenJObject
{
{ "Data", attachmentData },
{ "Metadata", item.Value <RavenJObject>("Metadata") },
{ "Key", item.Value <string>("Key") }
}
.WriteTo(jsonWriter);
}
lastEtag = new Guid(attachmentInfo.Last().Value <string>("Etag"));
}
}
protected override Guid ExportAttachments(JsonTextWriter jsonWriter, Guid lastEtag)
{
int totalCount = 0;
while (true)
{
RavenJArray attachmentInfo = null;
var request = CreateRequest("/static/?pageSize=" + smugglerOptions.BatchSize + "&etag=" + lastEtag);
request.ExecuteRequest(reader => attachmentInfo = RavenJArray.Load(new JsonTextReader(reader)));
if (attachmentInfo.Length == 0)
{
ShowProgress("Done with reading attachments, total: {0}", totalCount);
return(lastEtag);
}
totalCount += attachmentInfo.Length;
ShowProgress("Reading batch of {0,3} attachments, read so far: {1,10:#,#;;0}", attachmentInfo.Length, totalCount);
foreach (var item in attachmentInfo)
{
ShowProgress("Downloading attachment: {0}", item.Value <string>("Key"));
byte[] attachmentData = null;
var requestData = CreateRequest("/static/" + item.Value <string>("Key"));
requestData.ExecuteRequest(reader => attachmentData = reader.ReadData());
new RavenJObject
{
{ "Data", attachmentData },
{ "Metadata", item.Value <RavenJObject>("Metadata") },
{ "Key", item.Value <string>("Key") }
}
.WriteTo(jsonWriter);
}
lastEtag = new Guid(attachmentInfo.Last().Value <string>("Etag"));
}
}
private void ExportIndexes(JsonTextWriter jsonWriter)
{
int totalCount = 0;
while (true)
{
RavenJArray indexes = null;
var request = CreateRequest("/indexes?pageSize=128&start=" + totalCount);
request.ExecuteRequest(reader => indexes = RavenJArray.Load(new JsonTextReader(reader)));
if (indexes.Length == 0)
{
Console.WriteLine("Done with reading indexes, total: {0}", totalCount);
break;
}
totalCount += indexes.Length;
Console.WriteLine("Reading batch of {0,3} indexes, read so far: {1,10:#,#;;0}", indexes.Length, totalCount);
foreach (RavenJToken item in indexes)
{
item.WriteTo(jsonWriter);
}
}
}
protected override RavenJArray GetDocuments(Guid lastEtag)
{
int retries = retriesCount;
while (true)
{
try
{
RavenJArray documents = null;
var request = CreateRequest("/docs?pageSize=" + smugglerOptions.BatchSize + "&etag=" + lastEtag);
request.ExecuteRequest(reader => documents = RavenJArray.Load(new JsonTextReader(reader)));
return(documents);
}
catch (Exception e)
{
if (retries-- == 0)
{
throw;
}
LastRequestErrored = true;
ShowProgress("Error reading from database, remaining attempts {0}, will retry. Error: {1}", retries, e, retriesCount);
}
}
}
public static RavenJArray ReadJsonArray(this IHttpContext context)
{
using (var streamReader = new StreamReader(context.Request.InputStream, GetRequestEncoding(context)))
using (var jsonReader = new RavenJsonTextReader(streamReader))
return(RavenJArray.Load(jsonReader));
}
protected override async Task <Etag> ExportAttachments(RavenConnectionStringOptions src, JsonTextWriter jsonWriter, Etag lastEtag, Etag maxEtag)
{
if (maxEtag != null)
{
throw new ArgumentException("We don't support maxEtag in SmugglerApi", maxEtag);
}
var totalCount = 0;
while (true)
{
try
{
if (SmugglerOptions.Limit - totalCount <= 0)
{
ShowProgress("Done with reading attachments, total: {0}", totalCount);
return(lastEtag);
}
var maxRecords = Math.Min(SmugglerOptions.Limit - totalCount, SmugglerOptions.BatchSize);
RavenJArray attachmentInfo = null;
var request = CreateRequest(src, "/static/?pageSize=" + maxRecords + "&etag=" + lastEtag);
request.ExecuteRequest(reader => attachmentInfo = RavenJArray.Load(new JsonTextReader(reader)));
if (attachmentInfo.Length == 0)
{
var databaseStatistics = await GetStats();
var lastEtagComparable = new ComparableByteArray(lastEtag);
if (lastEtagComparable.CompareTo(databaseStatistics.LastAttachmentEtag) < 0)
{
lastEtag = EtagUtil.Increment(lastEtag, maxRecords);
ShowProgress("Got no results but didn't get to the last attachment etag, trying from: {0}", lastEtag);
continue;
}
ShowProgress("Done with reading attachments, total: {0}", totalCount);
return(lastEtag);
}
ShowProgress("Reading batch of {0,3} attachments, read so far: {1,10:#,#;;0}", attachmentInfo.Length,
totalCount);
foreach (var item in attachmentInfo)
{
ShowProgress("Downloading attachment: {0}", item.Value <string>("Key"));
byte[] attachmentData = null;
var requestData = CreateRequest(src, "/static/" + item.Value <string>("Key"));
requestData.ExecuteRequest(reader => attachmentData = reader.ReadData());
new RavenJObject
{
{ "Data", attachmentData },
{ "Metadata", item.Value <RavenJObject>("Metadata") },
{ "Key", item.Value <string>("Key") }
}
.WriteTo(jsonWriter);
totalCount++;
lastEtag = Etag.Parse(item.Value <string>("Etag"));
}
}
catch (Exception e)
{
ShowProgress("Got Exception during smuggler export. Exception: {0}. ", e.Message);
ShowProgress("Done with reading attachments, total: {0}", totalCount, lastEtag);
throw new SmugglerExportException(e.Message, e)
{
LastEtag = lastEtag,
};
}
}
}
