protected override Task <IAsyncEnumerator <RavenJObject> > GetDocuments(Etag lastEtag) { if (IsDocsStreamingSupported) { ShowProgress("Streaming documents from " + lastEtag); return(Commands.StreamDocsAsync(lastEtag)); } int retries = RetriesCount; while (true) { try { RavenJArray documents = null; var url = "/docs?pageSize=" + SmugglerOptions.BatchSize + "&etag=" + lastEtag; ShowProgress("GET " + url); var request = CreateRequest(url); request.ExecuteRequest(reader => documents = RavenJArray.Load(new JsonTextReader(reader))); return(new CompletedTask <IAsyncEnumerator <RavenJObject> >(new AsyncEnumeratorBridge <RavenJObject>(documents.Values <RavenJObject>().GetEnumerator()))); } catch (Exception e) { if (retries-- == 0) { throw; } LastRequestErrored = true; ShowProgress("Error reading from database, remaining attempts {0}, will retry. Error: {1}", retries, e, RetriesCount); } } }
public async Task <RavenJArray> ReadJsonArrayAsync() { using (var stream = await InnerRequest.Content.ReadAsStreamAsync()) using (var streamReader = new StreamReader(stream, GetRequestEncoding())) using (var jsonReader = new RavenJsonTextReader(streamReader)) return(RavenJArray.Load(jsonReader)); }
protected async override Task <IAsyncEnumerator <RavenJObject> > GetDocuments(RavenConnectionStringOptions src, Etag lastEtag, int take) { if (IsDocsStreamingSupported) { ShowProgress("Streaming documents from {0}, batch size {1}", lastEtag, take); return(await Commands.StreamDocsAsync(lastEtag, pageSize : take)); } int retries = RetriesCount; while (true) { try { RavenJArray documents = null; var url = "/docs?pageSize=" + Math.Min(SmugglerOptions.BatchSize, take) + "&etag=" + lastEtag; ShowProgress("GET " + url); var request = CreateRequest(src, url); request.ExecuteRequest(reader => documents = RavenJArray.Load(new JsonTextReader(reader))); return(new AsyncEnumeratorBridge <RavenJObject>(documents.Values <RavenJObject>().GetEnumerator())); } catch (Exception e) { if (retries-- == 0) { throw; } LastRequestErrored = true; ShowProgress("Error reading from database, remaining attempts {0}, will retry. Error: {1}", retries, e); } } }
private void ExportDocuments(SmugglerOptions options, JsonTextWriter jsonWriter) { var lastEtag = Guid.Empty; int totalCount = 0; while (true) { RavenJArray documents = null; var request = CreateRequest("/docs?pageSize=128&etag=" + lastEtag); request.ExecuteRequest(reader => documents = RavenJArray.Load(new JsonTextReader(reader))); if (documents.Length == 0) { Console.WriteLine("Done with reading documents, total: {0}", totalCount); break; } var final = documents.Where(options.MatchFilters).ToList(); final.ForEach(item => item.WriteTo(jsonWriter)); totalCount += final.Count; Console.WriteLine("Reading batch of {0,3} documents, read so far: {1,10:#,#;;0}", documents.Length, totalCount); lastEtag = new Guid(documents.Last().Value <RavenJObject>("@metadata").Value <string>("@etag")); } }
protected override RavenJArray GetDocuments(Guid lastEtag) { RavenJArray documents = null; var request = CreateRequest("/docs?pageSize=128&etag=" + lastEtag); request.ExecuteRequest(reader => documents = RavenJArray.Load(new JsonTextReader(reader))); return(documents); }
protected override RavenJArray GetIndexes(int totalCount) { RavenJArray indexes = null; var request = CreateRequest("/indexes?pageSize=128&start=" + totalCount); request.ExecuteRequest(reader => indexes = RavenJArray.Load(new JsonTextReader(reader))); return(indexes); }
protected override Task <RavenJArray> GetIndexes(int totalCount) { RavenJArray indexes = null; var request = CreateRequest("/indexes?pageSize=" + SmugglerOptions.BatchSize + "&start=" + totalCount); request.ExecuteRequest(reader => indexes = RavenJArray.Load(new JsonTextReader(reader))); return(new CompletedTask <RavenJArray>(indexes)); }
protected async override Task <RavenJArray> GetIndexes(RavenConnectionStringOptions src, int totalCount) { RavenJArray indexes = null; var request = CreateRequest(src, "/indexes?pageSize=" + SmugglerOptions.BatchSize + "&start=" + totalCount); request.ExecuteRequest(reader => indexes = RavenJArray.Load(new JsonTextReader(reader))); return(indexes); }
protected async Task <RavenJArray> ReadJsonArrayAsync() { using (var stream = await InnerRequest.Content.ReadAsStreamAsync().ConfigureAwait(false)) using (var buffered = new BufferedStream(stream)) using (var streamReader = new StreamReader(buffered, GetRequestEncoding())) using (var jsonReader = new RavenJsonTextReader(streamReader)) { return(RavenJArray.Load(jsonReader)); } }
protected override Task <RavenJArray> GetTransformers(int start) { if (IsTransformersSupported == false) { return(new CompletedTask <RavenJArray>(new RavenJArray())); } RavenJArray transformers = null; var request = CreateRequest("/transformers?pageSize=" + SmugglerOptions.BatchSize + "&start=" + start); request.ExecuteRequest(reader => transformers = RavenJArray.Load(new JsonTextReader(reader))); return(new CompletedTask <RavenJArray>(transformers)); }
protected async override Task <RavenJArray> GetTransformers(RavenConnectionStringOptions src, int start) { if (IsTransformersSupported == false) { return(new RavenJArray()); } RavenJArray transformers = null; var request = CreateRequest(src, "/transformers?pageSize=" + SmugglerOptions.BatchSize + "&start=" + start); request.ExecuteRequest(reader => transformers = RavenJArray.Load(new JsonTextReader(reader))); return(transformers); }
protected override Task <RavenJArray> GetTransformers(int totalCount) { if (Mode == SmugglerMode.Legacy) { return(new CompletedTask <RavenJArray>(new RavenJArray())); } RavenJArray transformers = null; var request = CreateRequest("/transformers?pageSize=" + SmugglerOptions.BatchSize + "&start=" + totalCount); request.ExecuteRequest(reader => transformers = RavenJArray.Load(new JsonTextReader(reader))); return(new CompletedTask <RavenJArray>(transformers)); }
///<summary> /// Get the possible terms for the specified field in the index asynchronously /// You can page through the results by use fromValue parameter as the /// starting point for the next query ///</summary> ///<returns></returns> public Task <string[]> GetTermsAsync(string index, string field, string fromValue, int pageSize) { return(url.Terms(index, field, fromValue, pageSize) .NoCache() .ToJsonRequest(this, credentials, convention) .ReadResponseStringAsync() .ContinueWith(task => { using (var reader = new JsonTextReader(new StringReader(task.Result))) { var json = RavenJArray.Load(reader); return json.Select(x => x.Value <string>()).ToArray(); } })); }
protected override async Task <Etag> ExportAttachments(JsonTextWriter jsonWriter, Etag lastEtag) { int totalCount = 0; while (true) { RavenJArray attachmentInfo = null; var request = CreateRequest("/static/?pageSize=" + SmugglerOptions.BatchSize + "&etag=" + lastEtag); request.ExecuteRequest(reader => attachmentInfo = RavenJArray.Load(new JsonTextReader(reader))); if (attachmentInfo.Length == 0) { var databaseStatistics = await GetStats(); var lastEtagComparable = new ComparableByteArray(lastEtag); if (lastEtagComparable.CompareTo(databaseStatistics.LastAttachmentEtag) < 0) { lastEtag = EtagUtil.Increment(lastEtag, SmugglerOptions.BatchSize); ShowProgress("Got no results but didn't get to the last attachment etag, trying from: {0}", lastEtag); continue; } ShowProgress("Done with reading attachments, total: {0}", totalCount); return(lastEtag); } totalCount += attachmentInfo.Length; ShowProgress("Reading batch of {0,3} attachments, read so far: {1,10:#,#;;0}", attachmentInfo.Length, totalCount); foreach (var item in attachmentInfo) { ShowProgress("Downloading attachment: {0}", item.Value <string>("Key")); byte[] attachmentData = null; var requestData = CreateRequest("/static/" + item.Value <string>("Key")); requestData.ExecuteRequest(reader => attachmentData = reader.ReadData()); new RavenJObject { { "Data", attachmentData }, { "Metadata", item.Value <RavenJObject>("Metadata") }, { "Key", item.Value <string>("Key") } } .WriteTo(jsonWriter); } lastEtag = Etag.Parse(attachmentInfo.Last().Value <string>("Etag")); } }
private void ExportAttachments(JsonTextWriter jsonWriter) { var lastEtag = Guid.Empty; int totalCount = 0; while (true) { RavenJArray attachmentInfo = null; var request = CreateRequest("/static/?pageSize=128&etag=" + lastEtag); request.ExecuteRequest(reader => attachmentInfo = RavenJArray.Load(new JsonTextReader(reader))); if (attachmentInfo.Length == 0) { Console.WriteLine("Done with reading attachments, total: {0}", totalCount); break; } totalCount += attachmentInfo.Length; Console.WriteLine("Reading batch of {0,3} attachments, read so far: {1,10:#,#;;0}", attachmentInfo.Length, totalCount); foreach (var item in attachmentInfo) { Console.WriteLine("Downloading attachment: {0}", item.Value <string>("Key")); byte[] attachmentData = null; var requestData = CreateRequest("/static/" + item.Value <string>("Key")); requestData.ExecuteRequest(reader => attachmentData = reader.ReadData()); new RavenJObject { { "Data", attachmentData }, { "Metadata", item.Value <RavenJObject>("Metadata") }, { "Key", item.Value <string>("Key") } } .WriteTo(jsonWriter); } lastEtag = new Guid(attachmentInfo.Last().Value <string>("Etag")); } }
protected override Guid ExportAttachments(JsonTextWriter jsonWriter, Guid lastEtag) { int totalCount = 0; while (true) { RavenJArray attachmentInfo = null; var request = CreateRequest("/static/?pageSize=" + smugglerOptions.BatchSize + "&etag=" + lastEtag); request.ExecuteRequest(reader => attachmentInfo = RavenJArray.Load(new JsonTextReader(reader))); if (attachmentInfo.Length == 0) { ShowProgress("Done with reading attachments, total: {0}", totalCount); return(lastEtag); } totalCount += attachmentInfo.Length; ShowProgress("Reading batch of {0,3} attachments, read so far: {1,10:#,#;;0}", attachmentInfo.Length, totalCount); foreach (var item in attachmentInfo) { ShowProgress("Downloading attachment: {0}", item.Value <string>("Key")); byte[] attachmentData = null; var requestData = CreateRequest("/static/" + item.Value <string>("Key")); requestData.ExecuteRequest(reader => attachmentData = reader.ReadData()); new RavenJObject { { "Data", attachmentData }, { "Metadata", item.Value <RavenJObject>("Metadata") }, { "Key", item.Value <string>("Key") } } .WriteTo(jsonWriter); } lastEtag = new Guid(attachmentInfo.Last().Value <string>("Etag")); } }
private void ExportIndexes(JsonTextWriter jsonWriter) { int totalCount = 0; while (true) { RavenJArray indexes = null; var request = CreateRequest("/indexes?pageSize=128&start=" + totalCount); request.ExecuteRequest(reader => indexes = RavenJArray.Load(new JsonTextReader(reader))); if (indexes.Length == 0) { Console.WriteLine("Done with reading indexes, total: {0}", totalCount); break; } totalCount += indexes.Length; Console.WriteLine("Reading batch of {0,3} indexes, read so far: {1,10:#,#;;0}", indexes.Length, totalCount); foreach (RavenJToken item in indexes) { item.WriteTo(jsonWriter); } } }
protected override RavenJArray GetDocuments(Guid lastEtag) { int retries = retriesCount; while (true) { try { RavenJArray documents = null; var request = CreateRequest("/docs?pageSize=" + smugglerOptions.BatchSize + "&etag=" + lastEtag); request.ExecuteRequest(reader => documents = RavenJArray.Load(new JsonTextReader(reader))); return(documents); } catch (Exception e) { if (retries-- == 0) { throw; } LastRequestErrored = true; ShowProgress("Error reading from database, remaining attempts {0}, will retry. Error: {1}", retries, e, retriesCount); } } }
public static RavenJArray ReadJsonArray(this IHttpContext context) { using (var streamReader = new StreamReader(context.Request.InputStream, GetRequestEncoding(context))) using (var jsonReader = new RavenJsonTextReader(streamReader)) return(RavenJArray.Load(jsonReader)); }
protected override async Task <Etag> ExportAttachments(RavenConnectionStringOptions src, JsonTextWriter jsonWriter, Etag lastEtag, Etag maxEtag) { if (maxEtag != null) { throw new ArgumentException("We don't support maxEtag in SmugglerApi", maxEtag); } var totalCount = 0; while (true) { try { if (SmugglerOptions.Limit - totalCount <= 0) { ShowProgress("Done with reading attachments, total: {0}", totalCount); return(lastEtag); } var maxRecords = Math.Min(SmugglerOptions.Limit - totalCount, SmugglerOptions.BatchSize); RavenJArray attachmentInfo = null; var request = CreateRequest(src, "/static/?pageSize=" + maxRecords + "&etag=" + lastEtag); request.ExecuteRequest(reader => attachmentInfo = RavenJArray.Load(new JsonTextReader(reader))); if (attachmentInfo.Length == 0) { var databaseStatistics = await GetStats(); var lastEtagComparable = new ComparableByteArray(lastEtag); if (lastEtagComparable.CompareTo(databaseStatistics.LastAttachmentEtag) < 0) { lastEtag = EtagUtil.Increment(lastEtag, maxRecords); ShowProgress("Got no results but didn't get to the last attachment etag, trying from: {0}", lastEtag); continue; } ShowProgress("Done with reading attachments, total: {0}", totalCount); return(lastEtag); } ShowProgress("Reading batch of {0,3} attachments, read so far: {1,10:#,#;;0}", attachmentInfo.Length, totalCount); foreach (var item in attachmentInfo) { ShowProgress("Downloading attachment: {0}", item.Value <string>("Key")); byte[] attachmentData = null; var requestData = CreateRequest(src, "/static/" + item.Value <string>("Key")); requestData.ExecuteRequest(reader => attachmentData = reader.ReadData()); new RavenJObject { { "Data", attachmentData }, { "Metadata", item.Value <RavenJObject>("Metadata") }, { "Key", item.Value <string>("Key") } } .WriteTo(jsonWriter); totalCount++; lastEtag = Etag.Parse(item.Value <string>("Etag")); } } catch (Exception e) { ShowProgress("Got Exception during smuggler export. Exception: {0}. ", e.Message); ShowProgress("Done with reading attachments, total: {0}", totalCount, lastEtag); throw new SmugglerExportException(e.Message, e) { LastEtag = lastEtag, }; } } }