protected override Task SerializeToStreamAsync(Stream stream, TransportContext context)
{
using (queryOp)
using (accessor)
using (_timeout)
using (var writer = GetOutputWriter(req, stream))
{
outputContentTypeSetter(writer.ContentType);
writer.WriteHeader();
try
{
queryOp.Execute(o =>
{
_timeout.Delay();
writer.Write(o);
});
}
catch (Exception e)
{
writer.WriteError(e);
}
}
return(Task.FromResult(true));
}
protected override Task SerializeToStreamAsync(Stream stream, TransportContext context)
{
var bufferSize = queryOp.Header.TotalResults > 1024 ? 1024 * 64 : 1024 * 8;
using (var bufferedStream = new BufferedStream(stream, bufferSize))
using (queryOp)
using (accessor)
using (_timeout)
using (var writer = GetOutputWriter(req, bufferedStream))
// we may be sending a LOT of documents to the user, and most
// of them aren't going to be relevant for other ops, so we are going to skip
// the cache for that, to avoid filling it up very quickly
using (DocumentCacher.SkipSettingDocumentsInDocumentCache())
{
outputContentTypeSetter(writer.ContentType);
writer.WriteHeader();
try
{
queryOp.Execute(o =>
{
_timeout.Delay();
writer.Write(o);
});
}
catch (Exception e)
{
writer.WriteError(e);
}
}
return(Task.FromResult(true));
}
private RavenJArray PerformBulkOperation(string index, IndexQuery indexQuery, bool allowStale, Func <string, TransactionInformation, object> batchOperation)
{
var array = new RavenJArray();
var bulkIndexQuery = new IndexQuery
{
Query = indexQuery.Query,
Start = indexQuery.Start,
Cutoff = indexQuery.Cutoff,
WaitForNonStaleResultsAsOfNow = indexQuery.WaitForNonStaleResultsAsOfNow,
PageSize = int.MaxValue,
FieldsToFetch = new[] { Constants.DocumentIdFieldName },
SortedFields = indexQuery.SortedFields,
HighlighterPreTags = indexQuery.HighlighterPreTags,
HighlighterPostTags = indexQuery.HighlighterPostTags,
HighlightedFields = indexQuery.HighlightedFields,
SortHints = indexQuery.SortHints
};
bool stale;
var queryResults = database.Queries.QueryDocumentIds(index, bulkIndexQuery, tokenSource, out stale);
if (stale && allowStale == false)
{
throw new InvalidOperationException(
"Bulk operation cancelled because the index is stale and allowStale is false");
}
var token = tokenSource.Token;
const int batchSize = 1024;
using (var enumerator = queryResults.GetEnumerator())
{
while (true)
{
if (timeout != null)
{
timeout.Delay();
}
var batchCount = 0;
token.ThrowIfCancellationRequested();
database.TransactionalStorage.Batch(actions =>
{
while (batchCount < batchSize && enumerator.MoveNext())
{
batchCount++;
var result = batchOperation(enumerator.Current, transactionInformation);
array.Add(RavenJObject.FromObject(result));
}
});
if (batchCount < batchSize)
{
break;
}
}
}
return(array);
}
private IEnumerable <JsonDocument> YieldDocumentsInBatch(CancellationTimeout timeout, Stream partialStream, Action <int> increaseDocumentsCount)
{
using (var stream = new GZipStream(partialStream, CompressionMode.Decompress, leaveOpen: true))
{
var reader = new BinaryReader(stream);
var count = reader.ReadInt32();
for (var i = 0; i < count; i++)
{
timeout.Delay();
var doc = (RavenJObject)RavenJToken.ReadFrom(new BsonReader(reader)
{
DateTimeKindHandling = DateTimeKind.Unspecified
});
var metadata = doc.Value <RavenJObject>("@metadata");
if (metadata == null)
{
throw new InvalidOperationException("Could not find metadata for document");
}
var id = metadata.Value <string>("@id");
if (string.IsNullOrEmpty(id))
{
throw new InvalidOperationException("Could not get id from metadata");
}
if (id.Equals(Constants.BulkImportHeartbeatDocKey, StringComparison.InvariantCultureIgnoreCase))
{
continue; //its just a token document, should not get written into the database
}
//the purpose of the heartbeat document is to make sure that the connection doesn't time-out
//during long pauses in the bulk insert operation.
// Currently used by smuggler to make sure that the connection doesn't time out if there is a
//continuation token and lots of document skips
doc.Remove("@metadata");
yield return(new JsonDocument
{
Key = id,
DataAsJson = doc,
Metadata = metadata
});
}
increaseDocumentsCount(count);
}
}
private IEnumerable <JsonDocument> YieldDocumentsInBatch(CancellationTimeout timeout, Stream partialStream, Action <int> increaseDocumentsCount)
{
using (var stream = new GZipStream(partialStream, CompressionMode.Decompress, leaveOpen: true))
{
var reader = new BinaryReader(stream);
var count = reader.ReadInt32();
for (var i = 0; i < count; i++)
{
timeout.Delay();
var doc = (RavenJObject)RavenJToken.ReadFrom(new BsonReader(reader)
{
DateTimeKindHandling = DateTimeKind.Unspecified
});
var metadata = doc.Value <RavenJObject>("@metadata");
if (metadata == null)
{
throw new InvalidOperationException("Could not find metadata for document");
}
var id = metadata.Value <string>("@id");
if (string.IsNullOrEmpty(id))
{
throw new InvalidOperationException("Could not get id from metadata");
}
doc.Remove("@metadata");
yield return(new JsonDocument
{
Key = id,
DataAsJson = doc,
Metadata = metadata
});
}
increaseDocumentsCount(count);
}
}
private IEnumerable <TimeSeriesAppend> YieldBatchItems(Stream partialStream, JsonSerializer serializer, CancellationTimeout timeout, Action <int> changeTimeSeriesFunc)
{
using (var stream = new GZipStream(partialStream, CompressionMode.Decompress, leaveOpen: true))
{
var reader = new BinaryReader(stream);
var count = reader.ReadInt32();
for (var i = 0; i < count; i++)
{
timeout.Delay();
var doc = (RavenJObject)RavenJToken.ReadFrom(new BsonReader(reader)
{
DateTimeKindHandling = DateTimeKind.Unspecified
});
yield return(doc.ToObject <TimeSeriesAppend>(serializer));
}
changeTimeSeriesFunc(count);
}
}
private IEnumerable <JsonDocument> YieldJsonDocumentsInBatch(CancellationTimeout timeout, Stream stream, int count, Action <int> increaseDocumentsCount)
{
using (JsonTextReader jsonReader = new JsonTextReader(new StreamReader(stream))
{
SupportMultipleContent = true
})
{
for (var i = 0; i < count; i++)
{
timeout.Delay();
while (jsonReader.Read())
{
if (jsonReader.TokenType == JsonToken.StartObject)
{
break;
}
}
if (jsonReader.TokenType != JsonToken.StartObject)
{
throw new InvalidOperationException("Could not get document");
}
var doc = (RavenJObject)RavenJToken.ReadFrom(jsonReader);
var jsonDocument = PrepareJsonDocument(doc);
if (jsonDocument == null)
{
continue;
}
yield return(jsonDocument);
}
increaseDocumentsCount(count);
}
}
private IEnumerable <JsonDocument> YieldBsonDocumentsInBatch(CancellationTimeout timeout, BinaryReader reader, int count, Action <int> increaseDocumentsCount)
{
using (var jsonReader = new BsonReader(reader)
{
SupportMultipleContent = true, DateTimeKindHandling = DateTimeKind.Unspecified
})
{
for (var i = 0; i < count; i++)
{
timeout.Delay();
while (jsonReader.Read())
{
if (jsonReader.TokenType == JsonToken.StartObject)
{
break;
}
}
if (jsonReader.TokenType != JsonToken.StartObject)
{
throw new InvalidOperationException("Could not get document");
}
var doc = (RavenJObject)RavenJToken.ReadFrom(jsonReader);
var jsonDocument = PrepareJsonDocument(doc);
if (jsonDocument == null)
{
continue;
}
yield return(jsonDocument);
}
increaseDocumentsCount(count);
}
}
private RavenJArray PerformBulkOperation(string index, IndexQuery indexQuery, BulkOperationOptions options, Func <string, TransactionInformation, object> batchOperation)
{
options = options ?? new BulkOperationOptions();
var array = new RavenJArray();
var bulkIndexQuery = new IndexQuery
{
Query = indexQuery.Query,
Start = indexQuery.Start,
Cutoff = indexQuery.Cutoff ?? SystemTime.UtcNow,
WaitForNonStaleResultsAsOfNow = indexQuery.WaitForNonStaleResultsAsOfNow,
PageSize = int.MaxValue,
FieldsToFetch = new[] { Constants.DocumentIdFieldName },
SortedFields = indexQuery.SortedFields,
HighlighterPreTags = indexQuery.HighlighterPreTags,
HighlighterPostTags = indexQuery.HighlighterPostTags,
HighlightedFields = indexQuery.HighlightedFields,
SortHints = indexQuery.SortHints
};
bool stale;
var queryResults = database.Queries.QueryDocumentIds(index, bulkIndexQuery, tokenSource, out stale);
if (stale && options.AllowStale == false)
{
if (options.StaleTimeout != null)
{
var staleWaitTimeout = Stopwatch.StartNew();
while (stale && staleWaitTimeout.Elapsed < options.StaleTimeout)
{
queryResults = database.Queries.QueryDocumentIds(index, bulkIndexQuery, tokenSource, out stale);
if (stale)
{
SystemTime.Wait(100);
}
}
}
if (stale)
{
if (options.StaleTimeout != null)
{
throw new InvalidOperationException("Bulk operation cancelled because the index is stale and StaleTimout of " + options.StaleTimeout + "passed");
}
throw new InvalidOperationException("Bulk operation cancelled because the index is stale and allowStale is false");
}
}
var token = tokenSource.Token;
const int batchSize = 1024;
int maxOpsPerSec = options.MaxOpsPerSec ?? int.MaxValue;
using (var enumerator = queryResults.GetEnumerator())
{
var duration = Stopwatch.StartNew();
var operations = 0;
while (true)
{
database.WorkContext.UpdateFoundWork();
if (timeout != null)
{
timeout.Delay();
}
var batchCount = 0;
var shouldWaitNow = false;
token.ThrowIfCancellationRequested();
using (database.DocumentLock.Lock())
{
database.TransactionalStorage.Batch(actions =>
{
while (batchCount < batchSize && enumerator.MoveNext())
{
batchCount++;
operations++;
var result = batchOperation(enumerator.Current, transactionInformation);
if (options.RetrieveDetails)
{
array.Add(RavenJObject.FromObject(result));
}
if (operations >= maxOpsPerSec && duration.ElapsedMilliseconds < 1000)
{
shouldWaitNow = true;
break;
}
}
});
if (shouldWaitNow)
{
SystemTime.Wait(500);
operations = 0;
duration.Restart();
continue;
}
}
if (batchCount < batchSize)
{
break;
}
}
}
return(array);
}
private static void WriteToStream(JsonWriter writer, RavenJObject item, CancellationTimeout timeout)
{
timeout.Delay();
item.WriteTo(writer);
}
