protected override Task SerializeToStreamAsync(Stream stream, TransportContext context) { using (queryOp) using (accessor) using (_timeout) using (var writer = GetOutputWriter(req, stream)) { outputContentTypeSetter(writer.ContentType); writer.WriteHeader(); try { queryOp.Execute(o => { _timeout.Delay(); writer.Write(o); }); } catch (Exception e) { writer.WriteError(e); } } return(Task.FromResult(true)); }
protected override Task SerializeToStreamAsync(Stream stream, TransportContext context) { var bufferSize = queryOp.Header.TotalResults > 1024 ? 1024 * 64 : 1024 * 8; using (var bufferedStream = new BufferedStream(stream, bufferSize)) using (queryOp) using (accessor) using (_timeout) using (var writer = GetOutputWriter(req, bufferedStream)) // we may be sending a LOT of documents to the user, and most // of them aren't going to be relevant for other ops, so we are going to skip // the cache for that, to avoid filling it up very quickly using (DocumentCacher.SkipSettingDocumentsInDocumentCache()) { outputContentTypeSetter(writer.ContentType); writer.WriteHeader(); try { queryOp.Execute(o => { _timeout.Delay(); writer.Write(o); }); } catch (Exception e) { writer.WriteError(e); } } return(Task.FromResult(true)); }
private RavenJArray PerformBulkOperation(string index, IndexQuery indexQuery, bool allowStale, Func <string, TransactionInformation, object> batchOperation) { var array = new RavenJArray(); var bulkIndexQuery = new IndexQuery { Query = indexQuery.Query, Start = indexQuery.Start, Cutoff = indexQuery.Cutoff, WaitForNonStaleResultsAsOfNow = indexQuery.WaitForNonStaleResultsAsOfNow, PageSize = int.MaxValue, FieldsToFetch = new[] { Constants.DocumentIdFieldName }, SortedFields = indexQuery.SortedFields, HighlighterPreTags = indexQuery.HighlighterPreTags, HighlighterPostTags = indexQuery.HighlighterPostTags, HighlightedFields = indexQuery.HighlightedFields, SortHints = indexQuery.SortHints }; bool stale; var queryResults = database.Queries.QueryDocumentIds(index, bulkIndexQuery, tokenSource, out stale); if (stale && allowStale == false) { throw new InvalidOperationException( "Bulk operation cancelled because the index is stale and allowStale is false"); } var token = tokenSource.Token; const int batchSize = 1024; using (var enumerator = queryResults.GetEnumerator()) { while (true) { if (timeout != null) { timeout.Delay(); } var batchCount = 0; token.ThrowIfCancellationRequested(); database.TransactionalStorage.Batch(actions => { while (batchCount < batchSize && enumerator.MoveNext()) { batchCount++; var result = batchOperation(enumerator.Current, transactionInformation); array.Add(RavenJObject.FromObject(result)); } }); if (batchCount < batchSize) { break; } } } return(array); }
private IEnumerable <JsonDocument> YieldDocumentsInBatch(CancellationTimeout timeout, Stream partialStream, Action <int> increaseDocumentsCount) { using (var stream = new GZipStream(partialStream, CompressionMode.Decompress, leaveOpen: true)) { var reader = new BinaryReader(stream); var count = reader.ReadInt32(); for (var i = 0; i < count; i++) { timeout.Delay(); var doc = (RavenJObject)RavenJToken.ReadFrom(new BsonReader(reader) { DateTimeKindHandling = DateTimeKind.Unspecified }); var metadata = doc.Value <RavenJObject>("@metadata"); if (metadata == null) { throw new InvalidOperationException("Could not find metadata for document"); } var id = metadata.Value <string>("@id"); if (string.IsNullOrEmpty(id)) { throw new InvalidOperationException("Could not get id from metadata"); } if (id.Equals(Constants.BulkImportHeartbeatDocKey, StringComparison.InvariantCultureIgnoreCase)) { continue; //its just a token document, should not get written into the database } //the purpose of the heartbeat document is to make sure that the connection doesn't time-out //during long pauses in the bulk insert operation. // Currently used by smuggler to make sure that the connection doesn't time out if there is a //continuation token and lots of document skips doc.Remove("@metadata"); yield return(new JsonDocument { Key = id, DataAsJson = doc, Metadata = metadata }); } increaseDocumentsCount(count); } }
private IEnumerable <JsonDocument> YieldDocumentsInBatch(CancellationTimeout timeout, Stream partialStream, Action <int> increaseDocumentsCount) { using (var stream = new GZipStream(partialStream, CompressionMode.Decompress, leaveOpen: true)) { var reader = new BinaryReader(stream); var count = reader.ReadInt32(); for (var i = 0; i < count; i++) { timeout.Delay(); var doc = (RavenJObject)RavenJToken.ReadFrom(new BsonReader(reader) { DateTimeKindHandling = DateTimeKind.Unspecified }); var metadata = doc.Value <RavenJObject>("@metadata"); if (metadata == null) { throw new InvalidOperationException("Could not find metadata for document"); } var id = metadata.Value <string>("@id"); if (string.IsNullOrEmpty(id)) { throw new InvalidOperationException("Could not get id from metadata"); } doc.Remove("@metadata"); yield return(new JsonDocument { Key = id, DataAsJson = doc, Metadata = metadata }); } increaseDocumentsCount(count); } }
private IEnumerable <TimeSeriesAppend> YieldBatchItems(Stream partialStream, JsonSerializer serializer, CancellationTimeout timeout, Action <int> changeTimeSeriesFunc) { using (var stream = new GZipStream(partialStream, CompressionMode.Decompress, leaveOpen: true)) { var reader = new BinaryReader(stream); var count = reader.ReadInt32(); for (var i = 0; i < count; i++) { timeout.Delay(); var doc = (RavenJObject)RavenJToken.ReadFrom(new BsonReader(reader) { DateTimeKindHandling = DateTimeKind.Unspecified }); yield return(doc.ToObject <TimeSeriesAppend>(serializer)); } changeTimeSeriesFunc(count); } }
private IEnumerable <JsonDocument> YieldJsonDocumentsInBatch(CancellationTimeout timeout, Stream stream, int count, Action <int> increaseDocumentsCount) { using (JsonTextReader jsonReader = new JsonTextReader(new StreamReader(stream)) { SupportMultipleContent = true }) { for (var i = 0; i < count; i++) { timeout.Delay(); while (jsonReader.Read()) { if (jsonReader.TokenType == JsonToken.StartObject) { break; } } if (jsonReader.TokenType != JsonToken.StartObject) { throw new InvalidOperationException("Could not get document"); } var doc = (RavenJObject)RavenJToken.ReadFrom(jsonReader); var jsonDocument = PrepareJsonDocument(doc); if (jsonDocument == null) { continue; } yield return(jsonDocument); } increaseDocumentsCount(count); } }
private IEnumerable <JsonDocument> YieldBsonDocumentsInBatch(CancellationTimeout timeout, BinaryReader reader, int count, Action <int> increaseDocumentsCount) { using (var jsonReader = new BsonReader(reader) { SupportMultipleContent = true, DateTimeKindHandling = DateTimeKind.Unspecified }) { for (var i = 0; i < count; i++) { timeout.Delay(); while (jsonReader.Read()) { if (jsonReader.TokenType == JsonToken.StartObject) { break; } } if (jsonReader.TokenType != JsonToken.StartObject) { throw new InvalidOperationException("Could not get document"); } var doc = (RavenJObject)RavenJToken.ReadFrom(jsonReader); var jsonDocument = PrepareJsonDocument(doc); if (jsonDocument == null) { continue; } yield return(jsonDocument); } increaseDocumentsCount(count); } }
private RavenJArray PerformBulkOperation(string index, IndexQuery indexQuery, BulkOperationOptions options, Func <string, TransactionInformation, object> batchOperation) { options = options ?? new BulkOperationOptions(); var array = new RavenJArray(); var bulkIndexQuery = new IndexQuery { Query = indexQuery.Query, Start = indexQuery.Start, Cutoff = indexQuery.Cutoff ?? SystemTime.UtcNow, WaitForNonStaleResultsAsOfNow = indexQuery.WaitForNonStaleResultsAsOfNow, PageSize = int.MaxValue, FieldsToFetch = new[] { Constants.DocumentIdFieldName }, SortedFields = indexQuery.SortedFields, HighlighterPreTags = indexQuery.HighlighterPreTags, HighlighterPostTags = indexQuery.HighlighterPostTags, HighlightedFields = indexQuery.HighlightedFields, SortHints = indexQuery.SortHints }; bool stale; var queryResults = database.Queries.QueryDocumentIds(index, bulkIndexQuery, tokenSource, out stale); if (stale && options.AllowStale == false) { if (options.StaleTimeout != null) { var staleWaitTimeout = Stopwatch.StartNew(); while (stale && staleWaitTimeout.Elapsed < options.StaleTimeout) { queryResults = database.Queries.QueryDocumentIds(index, bulkIndexQuery, tokenSource, out stale); if (stale) { SystemTime.Wait(100); } } } if (stale) { if (options.StaleTimeout != null) { throw new InvalidOperationException("Bulk operation cancelled because the index is stale and StaleTimout of " + options.StaleTimeout + "passed"); } throw new InvalidOperationException("Bulk operation cancelled because the index is stale and allowStale is false"); } } var token = tokenSource.Token; const int batchSize = 1024; int maxOpsPerSec = options.MaxOpsPerSec ?? int.MaxValue; using (var enumerator = queryResults.GetEnumerator()) { var duration = Stopwatch.StartNew(); var operations = 0; while (true) { database.WorkContext.UpdateFoundWork(); if (timeout != null) { timeout.Delay(); } var batchCount = 0; var shouldWaitNow = false; token.ThrowIfCancellationRequested(); using (database.DocumentLock.Lock()) { database.TransactionalStorage.Batch(actions => { while (batchCount < batchSize && enumerator.MoveNext()) { batchCount++; operations++; var result = batchOperation(enumerator.Current, transactionInformation); if (options.RetrieveDetails) { array.Add(RavenJObject.FromObject(result)); } if (operations >= maxOpsPerSec && duration.ElapsedMilliseconds < 1000) { shouldWaitNow = true; break; } } }); if (shouldWaitNow) { SystemTime.Wait(500); operations = 0; duration.Restart(); continue; } } if (batchCount < batchSize) { break; } } } return(array); }
private static void WriteToStream(JsonWriter writer, RavenJObject item, CancellationTimeout timeout) { timeout.Delay(); item.WriteTo(writer); }