public static void WriteMetadata(this BlittableJsonTextWriter writer, Document document, BlittableJsonReaderObject metadata)
{
writer.WritePropertyName(Constants.Documents.Metadata.Key);
writer.WriteStartObject();
bool first = true;
if (metadata != null)
{
var size = metadata.Count;
var prop = new BlittableJsonReaderObject.PropertyDetails();
for (int i = 0; i < size; i++)
{
if (first == false)
{
writer.WriteComma();
}
first = false;
metadata.GetPropertyByIndex(i, ref prop);
writer.WritePropertyName(prop.Name);
writer.WriteValue(prop.Token & BlittableJsonReaderBase.TypesMask, prop.Value);
}
}
if (first == false)
{
writer.WriteComma();
}
writer.WritePropertyName(Constants.Documents.Metadata.ChangeVector);
writer.WriteString(document.ChangeVector);
if (document.Flags != DocumentFlags.None)
{
writer.WriteComma();
writer.WritePropertyName(Constants.Documents.Metadata.Flags);
writer.WriteString(document.Flags.ToString());
}
if (document.Id != null)
{
writer.WriteComma();
writer.WritePropertyName(Constants.Documents.Metadata.Id);
writer.WriteString(document.Id);
}
if (document.IndexScore != null)
{
writer.WriteComma();
writer.WritePropertyName(Constants.Documents.Metadata.IndexScore);
writer.WriteDouble(document.IndexScore.Value);
}
if (document.LastModified != DateTime.MinValue)
{
writer.WriteComma();
writer.WritePropertyName(Constants.Documents.Metadata.LastModified);
writer.WriteDateTime(document.LastModified, isUtc: true);
}
writer.WriteEndObject();
}
private static void WriteDocumentProperties(this BlittableJsonTextWriter writer, JsonOperationContext context, Document document, Func <LazyStringValue, bool> filterMetadataProperty = null)
{
var first = true;
BlittableJsonReaderObject metadata = null;
var metadataField = context.GetLazyStringForFieldWithCaching(Raven.Server.Json.BlittableJsonTextWriterExtensions.MetadataKeySegment);
var prop = new BlittableJsonReaderObject.PropertyDetails();
using (var buffers = document.Data.GetPropertiesByInsertionOrder())
{
for (var i = 0; i < buffers.Size; i++)
{
unsafe
{
document.Data.GetPropertyByIndex(buffers.Properties[i], ref prop);
}
if (metadataField.Equals(prop.Name))
{
metadata = (BlittableJsonReaderObject)prop.Value;
continue;
}
if (first == false)
{
writer.WriteComma();
}
first = false;
writer.WritePropertyName((string)prop.Name);
writer.WriteValue(prop.Token & BlittableJsonReaderBase.TypesMask, prop.Value);
}
}
if (first == false)
{
writer.WriteComma();
}
WriteMetadata(writer, document, metadata, filterMetadataProperty);
}
private static void WriteDocumentProperties(this BlittableJsonTextWriter writer, JsonOperationContext context, Document document)
{
if (_buffers == null)
{
_buffers = new BlittableJsonReaderObject.PropertiesInsertionBuffer();
}
var first = true;
BlittableJsonReaderObject metadata = null;
var metadataField = context.GetLazyStringForFieldWithCaching(MetadataKeySegment);
var size = document.Data.GetPropertiesByInsertionOrder(_buffers);
var prop = new BlittableJsonReaderObject.PropertyDetails();
for (var i = 0; i < size; i++)
{
document.Data.GetPropertyByIndex(_buffers.Properties[i], ref prop);
if (metadataField.Equals(prop.Name))
{
metadata = (BlittableJsonReaderObject)prop.Value;
continue;
}
if (first == false)
{
writer.WriteComma();
}
first = false;
writer.WritePropertyName(prop.Name);
writer.WriteValue(prop.Token & BlittableJsonReaderBase.TypesMask, prop.Value);
}
if (first == false)
{
writer.WriteComma();
}
WriteMetadata(writer, document, metadata);
}
public static void WriteDocumentPropertiesWithoutMetdata(this BlittableJsonTextWriter writer, JsonOperationContext context, Document document)
{
if (_buffers == null)
{
_buffers = new BlittableJsonReaderObject.PropertiesInsertionBuffer();
}
var first = true;
var size = document.Data.GetPropertiesByInsertionOrder(_buffers);
var prop = new BlittableJsonReaderObject.PropertyDetails();
for (var i = 0; i < size; i++)
{
document.Data.GetPropertyByIndex(_buffers.Properties[i], ref prop);
if (first == false)
{
writer.WriteComma();
}
first = false;
writer.WritePropertyName(prop.Name);
writer.WriteValue(prop.Token & BlittableJsonReaderBase.TypesMask, prop.Value);
}
}
private async Task ProcessSubscriptionAsync()
{
if (_logger.IsInfoEnabled)
{
_logger.Info(
$"Starting processing documents for subscription {SubscriptionId} received from {TcpConnection.TcpClient.Client.RemoteEndPoint}");
}
using (DisposeOnDisconnect)
using (TcpConnection.DocumentDatabase.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext docsContext))
using (RegisterForNotificationOnNewDocuments())
{
var replyFromClientTask = GetReplyFromClientAsync();
string lastChangeVector = null;
string subscriptionChangeVectorBeforeCurrentBatch = SubscriptionState.ChangeVectorForNextBatchStartingPoint;
var startEtag = GetStartEtagForSubscription(docsContext, SubscriptionState);
var patch = SetupFilterScript();
var fetcher = new SubscriptionDocumentsFetcher(TcpConnection.DocumentDatabase, _options.MaxDocsPerBatch, SubscriptionId, TcpConnection.TcpClient.Client.RemoteEndPoint);
while (CancellationTokenSource.IsCancellationRequested == false)
{
bool anyDocumentsSentInCurrentIteration = false;
var sendingCurrentBatchStopwatch = Stopwatch.StartNew();
_buffer.SetLength(0);
var docsToFlush = 0;
using (TcpConnection.ContextPool.AllocateOperationContext(out JsonOperationContext context))
using (var writer = new BlittableJsonTextWriter(context, _buffer))
{
using (docsContext.OpenReadTransaction())
{
foreach (var result in fetcher.GetDataToSend(docsContext, Collection, Revisions, SubscriptionState, patch, startEtag))
{
startEtag = result.Doc.Etag;
lastChangeVector = string.IsNullOrEmpty(SubscriptionState.ChangeVectorForNextBatchStartingPoint)
? result.Doc.ChangeVector
: ChangeVectorUtils.MergeVectors(result.Doc.ChangeVector, SubscriptionState.ChangeVectorForNextBatchStartingPoint);
if (result.Doc.Data == null)
{
if (sendingCurrentBatchStopwatch.ElapsedMilliseconds > 1000)
{
await SendHeartBeat();
sendingCurrentBatchStopwatch.Restart();
}
continue;
}
anyDocumentsSentInCurrentIteration = true;
writer.WriteStartObject();
writer.WritePropertyName(context.GetLazyStringForFieldWithCaching(TypeSegment));
writer.WriteValue(BlittableJsonToken.String, context.GetLazyStringForFieldWithCaching(DataSegment));
writer.WriteComma();
writer.WritePropertyName(context.GetLazyStringForFieldWithCaching(DataSegment));
result.Doc.EnsureMetadata();
if (result.Exception != null)
{
var metadata = result.Doc.Data[Client.Constants.Documents.Metadata.Key];
writer.WriteValue(BlittableJsonToken.StartObject,
docsContext.ReadObject(new DynamicJsonValue
{
[Client.Constants.Documents.Metadata.Key] = metadata
}, result.Doc.Id)
);
writer.WriteComma();
writer.WritePropertyName(context.GetLazyStringForFieldWithCaching(ExceptionSegment));
writer.WriteValue(BlittableJsonToken.String, context.GetLazyStringForFieldWithCaching(result.Exception.ToString()));
}
else
{
writer.WriteDocument(docsContext, result.Doc, metadataOnly: false);
}
writer.WriteEndObject();
docsToFlush++;
// perform flush for current batch after 1000ms of running or 1 MB
if (_buffer.Length > Constants.Size.Megabyte ||
sendingCurrentBatchStopwatch.ElapsedMilliseconds > 1000)
{
if (docsToFlush > 0)
{
await FlushDocsToClient(writer, docsToFlush);
docsToFlush = 0;
sendingCurrentBatchStopwatch.Restart();
}
else
{
await SendHeartBeat();
}
}
}
}
if (anyDocumentsSentInCurrentIteration)
{
context.Write(writer, new DynamicJsonValue
{
[nameof(SubscriptionConnectionServerMessage.Type)] = nameof(SubscriptionConnectionServerMessage.MessageType.EndOfBatch)
});
await FlushDocsToClient(writer, docsToFlush, true);
if (_logger.IsInfoEnabled)
{
_logger.Info(
$"Finished sending a batch with {docsToFlush} documents for subscription {Options.SubscriptionName}");
}
}
}
if (anyDocumentsSentInCurrentIteration == false)
{
if (_logger.IsInfoEnabled)
{
_logger.Info(
$"Finished sending a batch with {docsToFlush} documents for subscription {Options.SubscriptionName}");
}
await TcpConnection.DocumentDatabase.SubscriptionStorage.AcknowledgeBatchProcessed(SubscriptionId,
Options.SubscriptionName,
lastChangeVector,
subscriptionChangeVectorBeforeCurrentBatch);
subscriptionChangeVectorBeforeCurrentBatch = lastChangeVector;
if (sendingCurrentBatchStopwatch.ElapsedMilliseconds > 1000)
{
await SendHeartBeat();
}
using (docsContext.OpenReadTransaction())
{
long globalEtag = TcpConnection.DocumentDatabase.DocumentsStorage.GetLastDocumentEtag(docsContext, Collection);
if (globalEtag > startEtag)
{
continue;
}
}
if (await WaitForChangedDocuments(replyFromClientTask))
{
continue;
}
}
SubscriptionConnectionClientMessage clientReply;
while (true)
{
var result = await Task.WhenAny(replyFromClientTask,
TimeoutManager.WaitFor(TimeSpan.FromMilliseconds(5000), CancellationTokenSource.Token)).ConfigureAwait(false);
CancellationTokenSource.Token.ThrowIfCancellationRequested();
if (result == replyFromClientTask)
{
clientReply = await replyFromClientTask;
if (clientReply.Type == SubscriptionConnectionClientMessage.MessageType.DisposedNotification)
{
CancellationTokenSource.Cancel();
break;
}
replyFromClientTask = GetReplyFromClientAsync();
break;
}
await SendHeartBeat();
}
CancellationTokenSource.Token.ThrowIfCancellationRequested();
switch (clientReply.Type)
{
case SubscriptionConnectionClientMessage.MessageType.Acknowledge:
await TcpConnection.DocumentDatabase.SubscriptionStorage.AcknowledgeBatchProcessed(
SubscriptionId,
Options.SubscriptionName,
lastChangeVector,
subscriptionChangeVectorBeforeCurrentBatch);
subscriptionChangeVectorBeforeCurrentBatch = lastChangeVector;
Stats.LastAckReceivedAt = DateTime.UtcNow;
Stats.AckRate.Mark();
await WriteJsonAsync(new DynamicJsonValue
{
[nameof(SubscriptionConnectionServerMessage.Type)] = nameof(SubscriptionConnectionServerMessage.MessageType.Confirm)
});
break;
//precaution, should not reach this case...
case SubscriptionConnectionClientMessage.MessageType.DisposedNotification:
CancellationTokenSource.Cancel();
break;
default:
throw new ArgumentException("Unknown message type from client " +
clientReply.Type);
}
}
CancellationTokenSource.Token.ThrowIfCancellationRequested();
}
}
private void WriteDocument(BlittableJsonTextWriter writer, JsonOperationContext context, Document document, HashSet <string> propertiesPreviewToSend, HashSet <string> fullPropertiesToSend)
{
if (_buffers == null)
{
_buffers = new BlittableJsonReaderObject.PropertiesInsertionBuffer();
}
writer.WriteStartObject();
document.Data.TryGet(Constants.Documents.Metadata.Key, out BlittableJsonReaderObject metadata);
bool first = true;
var objectsStubs = new HashSet <LazyStringValue>();
var arraysStubs = new HashSet <LazyStringValue>();
var trimmedValue = new HashSet <LazyStringValue>();
var size = document.Data.GetPropertiesByInsertionOrder(_buffers);
var prop = new BlittableJsonReaderObject.PropertyDetails();
for (int i = 0; i < size; i++)
{
document.Data.GetPropertyByIndex(_buffers.Properties[i], ref prop);
var sendFull = fullPropertiesToSend.Contains(prop.Name);
if (sendFull || propertiesPreviewToSend.Contains(prop.Name))
{
var strategy = sendFull ? ValueWriteStrategy.Passthrough : FindWriteStrategy(prop.Token & BlittableJsonReaderBase.TypesMask, prop.Value);
if (strategy == ValueWriteStrategy.Passthrough || strategy == ValueWriteStrategy.Trim)
{
if (first == false)
{
writer.WriteComma();
}
first = false;
}
switch (strategy)
{
case ValueWriteStrategy.Passthrough:
writer.WritePropertyName(prop.Name);
writer.WriteValue(prop.Token & BlittableJsonReaderBase.TypesMask, prop.Value);
break;
case ValueWriteStrategy.SubstituteWithArrayStub:
arraysStubs.Add(prop.Name);
break;
case ValueWriteStrategy.SubstituteWithObjectStub:
objectsStubs.Add(prop.Name);
break;
case ValueWriteStrategy.Trim:
writer.WritePropertyName(prop.Name);
WriteTrimmedValue(writer, prop.Token & BlittableJsonReaderBase.TypesMask, prop.Value);
trimmedValue.Add(prop.Name);
break;
}
}
}
if (first == false)
{
writer.WriteComma();
}
var extraMetadataProperties = new DynamicJsonValue
{
[ObjectStubsKey] = new DynamicJsonArray(objectsStubs),
[ArrayStubsKey] = new DynamicJsonArray(arraysStubs),
[TrimmedValueKey] = new DynamicJsonArray(trimmedValue)
};
if (metadata != null)
{
metadata.Modifications = extraMetadataProperties;
metadata = context.ReadObject(metadata, document.Id);
}
else
{
metadata = context.ReadObject(extraMetadataProperties, document.Id);
}
writer.WriteMetadata(document, metadata);
writer.WriteEndObject();
}
private static void WriteMetadata(BlittableJsonTextWriter writer, Document document, BlittableJsonReaderObject metadata, Func <LazyStringValue, bool> filterMetadataProperty = null)
{
writer.WritePropertyName(Constants.Documents.Metadata.Key);
writer.WriteStartObject();
bool first = true;
if (metadata != null)
{
var size = metadata.Count;
var prop = new BlittableJsonReaderObject.PropertyDetails();
for (int i = 0; i < size; i++)
{
metadata.GetPropertyByIndex(i, ref prop);
if (filterMetadataProperty != null && filterMetadataProperty(prop.Name))
{
continue;
}
if (first == false)
{
writer.WriteComma();
}
first = false;
writer.WritePropertyName((string)prop.Name);
writer.WriteValue(prop.Token & BlittableJsonReaderBase.TypesMask, prop.Value);
}
}
if (first == false)
{
writer.WriteComma();
}
writer.WritePropertyName(Constants.Documents.Metadata.ChangeVector);
writer.WriteString(document.ChangeVector);
if (document.Flags != DocumentFlags.None)
{
writer.WriteComma();
writer.WritePropertyName(Constants.Documents.Metadata.Flags);
writer.WriteString(document.Flags.ToString());
}
if (document.Id != null)
{
writer.WriteComma();
writer.WritePropertyName(Constants.Documents.Metadata.Id);
writer.WriteString(document.Id);
}
if (document.IndexScore != null)
{
writer.WriteComma();
writer.WritePropertyName(Constants.Documents.Metadata.IndexScore);
writer.WriteDouble(document.IndexScore.Value);
}
if (document.Distance != null)
{
writer.WriteComma();
var result = document.Distance.Value;
writer.WritePropertyName(Constants.Documents.Metadata.SpatialResult);
writer.WriteStartObject();
writer.WritePropertyName(nameof(result.Distance));
writer.WriteDouble(result.Distance);
writer.WriteComma();
writer.WritePropertyName(nameof(result.Latitude));
writer.WriteDouble(result.Latitude);
writer.WriteComma();
writer.WritePropertyName(nameof(result.Longitude));
writer.WriteDouble(result.Longitude);
writer.WriteEndObject();
}
if (document.LastModified != DateTime.MinValue)
{
writer.WriteComma();
writer.WritePropertyName(Constants.Documents.Metadata.LastModified);
writer.WriteDateTime(document.LastModified, isUtc: true);
}
writer.WriteEndObject();
}
/// <summary>
/// Iterates on a batch in document collection, process it and send documents if found any match
/// </summary>
/// <param name="docsContext"></param>
/// <param name="sendingCurrentBatchStopwatch"></param>
/// <returns>Whether succeeded finding any documents to send</returns>
private async Task <bool> TrySendingBatchToClient(DocumentsOperationContext docsContext, Stopwatch sendingCurrentBatchStopwatch)
{
bool anyDocumentsSentInCurrentIteration = false;
int docsToFlush = 0;
using (var writer = new BlittableJsonTextWriter(docsContext, _buffer))
{
using (docsContext.OpenReadTransaction())
{
foreach (var result in _documentsFetcher.GetDataToSend(docsContext, _startEtag))
{
_startEtag = result.Doc.Etag;
_lastChangeVector = string.IsNullOrEmpty(SubscriptionState.ChangeVectorForNextBatchStartingPoint)
? result.Doc.ChangeVector
: ChangeVectorUtils.MergeVectors(result.Doc.ChangeVector, SubscriptionState.ChangeVectorForNextBatchStartingPoint);
if (result.Doc.Data == null)
{
if (sendingCurrentBatchStopwatch.ElapsedMilliseconds > 1000)
{
await SendHeartBeat();
sendingCurrentBatchStopwatch.Restart();
}
continue;
}
anyDocumentsSentInCurrentIteration = true;
writer.WriteStartObject();
writer.WritePropertyName(docsContext.GetLazyStringForFieldWithCaching(TypeSegment));
writer.WriteValue(BlittableJsonToken.String, docsContext.GetLazyStringForFieldWithCaching(DataSegment));
writer.WriteComma();
writer.WritePropertyName(docsContext.GetLazyStringForFieldWithCaching(DataSegment));
result.Doc.EnsureMetadata();
if (result.Exception != null)
{
var metadata = result.Doc.Data[Client.Constants.Documents.Metadata.Key];
writer.WriteValue(BlittableJsonToken.StartObject,
docsContext.ReadObject(new DynamicJsonValue
{
[Client.Constants.Documents.Metadata.Key] = metadata
}, result.Doc.Id)
);
writer.WriteComma();
writer.WritePropertyName(docsContext.GetLazyStringForFieldWithCaching(ExceptionSegment));
writer.WriteValue(BlittableJsonToken.String, docsContext.GetLazyStringForFieldWithCaching(result.Exception.ToString()));
}
else
{
writer.WriteDocument(docsContext, result.Doc, metadataOnly: false);
}
writer.WriteEndObject();
docsToFlush++;
// perform flush for current batch after 1000ms of running or 1 MB
if (_buffer.Length > Constants.Size.Megabyte ||
sendingCurrentBatchStopwatch.ElapsedMilliseconds > 1000)
{
if (docsToFlush > 0)
{
await FlushDocsToClient(writer, docsToFlush);
docsToFlush = 0;
sendingCurrentBatchStopwatch.Restart();
}
else
{
await SendHeartBeat();
}
}
}
}
if (anyDocumentsSentInCurrentIteration)
{
docsContext.Write(writer, new DynamicJsonValue
{
[nameof(SubscriptionConnectionServerMessage.Type)] = nameof(SubscriptionConnectionServerMessage.MessageType.EndOfBatch)
});
await FlushDocsToClient(writer, docsToFlush, true);
if (_logger.IsInfoEnabled)
{
_logger.Info(
$"Finished sending a batch with {docsToFlush} documents for subscription {Options.SubscriptionName}");
}
}
}
return(anyDocumentsSentInCurrentIteration);
}
private unsafe void WriteDocument(BlittableJsonTextWriter writer, JsonOperationContext context, Document document, HashSet <string> propertiesPreviewToSend, HashSet <string> fullPropertiesToSend)
{
writer.WriteStartObject();
document.Data.TryGet(Constants.Documents.Metadata.Key, out BlittableJsonReaderObject metadata);
bool first = true;
var arrayStubsJson = new DynamicJsonValue();
var objectStubsJson = new DynamicJsonValue();
var trimmedValue = new HashSet <LazyStringValue>();
var prop = new BlittableJsonReaderObject.PropertyDetails();
using (var buffers = document.Data.GetPropertiesByInsertionOrder())
{
for (int i = 0; i < buffers.Size; i++)
{
document.Data.GetPropertyByIndex(buffers.Properties[i], ref prop);
var sendFull = fullPropertiesToSend.Contains(prop.Name);
if (sendFull || propertiesPreviewToSend.Contains(prop.Name))
{
var strategy = sendFull ? ValueWriteStrategy.Passthrough : FindWriteStrategy(prop.Token & BlittableJsonReaderBase.TypesMask, prop.Value);
if (strategy == ValueWriteStrategy.Passthrough || strategy == ValueWriteStrategy.Trim)
{
if (first == false)
{
writer.WriteComma();
}
first = false;
}
switch (strategy)
{
case ValueWriteStrategy.Passthrough:
writer.WritePropertyName(prop.Name);
writer.WriteValue(prop.Token & BlittableJsonReaderBase.TypesMask, prop.Value);
break;
case ValueWriteStrategy.SubstituteWithArrayStub:
arrayStubsJson[prop.Name] = ((BlittableJsonReaderArray)prop.Value).Length;
break;
case ValueWriteStrategy.SubstituteWithObjectStub:
objectStubsJson[prop.Name] = ((BlittableJsonReaderObject)prop.Value).Count;
break;
case ValueWriteStrategy.Trim:
writer.WritePropertyName(prop.Name);
WriteTrimmedValue(writer, prop.Token & BlittableJsonReaderBase.TypesMask, prop.Value);
trimmedValue.Add(prop.Name);
break;
}
}
}
}
if (first == false)
{
writer.WriteComma();
}
var extraMetadataProperties = new DynamicJsonValue(metadata)
{
[ObjectStubsKey] = objectStubsJson,
[ArrayStubsKey] = arrayStubsJson,
[TrimmedValueKey] = new DynamicJsonArray(trimmedValue)
};
if (metadata != null)
{
metadata.Modifications = extraMetadataProperties;
if (document.Flags.Contain(DocumentFlags.HasCounters) || document.Flags.Contain(DocumentFlags.HasAttachments) || document.Flags.Contain(DocumentFlags.HasTimeSeries))
{
metadata.Modifications.Remove(Constants.Documents.Metadata.Counters);
metadata.Modifications.Remove(Constants.Documents.Metadata.Attachments);
metadata.Modifications.Remove(Constants.Documents.Metadata.TimeSeries);
}
using (var old = metadata)
{
metadata = context.ReadObject(metadata, document.Id);
}
}
else
{
metadata = context.ReadObject(extraMetadataProperties, document.Id);
}
writer.WriteMetadata(document, metadata);
writer.WriteEndObject();
}
