public void AddResult(Document res)
{
if (_first == false)
{
_writer.WriteComma();
}
else
{
_first = false;
}
_writer.WriteDocument(_context, res, metadataOnly: false);
}
public override void AddResult(Document result)
{
if (_anyWrites == false)
{
StartResponseIfNeeded();
}
else
{
_writer.WriteComma();
}
_writer.WriteDocument(_context, result, metadataOnly: false);
}
public override void AddResult(Document result)
{
if (_anyWrites == false)
{
StartResponseIfNeeded();
}
else
{
_writer.WriteComma();
}
using (result.Data)
{
_writer.WriteDocument(_context, result);
}
}
public Task Get()
{
var name = RouteMatch.Url.Substring(RouteMatch.MatchLength);
if (string.IsNullOrWhiteSpace(name))
{
throw new InvalidOperationException("Database name was not provided");
}
TransactionOperationContext context;
using (ServerStore.ContextPool.AllocateOperationContext(out context))
{
context.OpenReadTransaction();
var dbId = Constants.Database.Prefix + name;
long etag;
var dbDoc = ServerStore.Read(context, dbId, out etag);
if (dbDoc == null)
{
HttpContext.Response.StatusCode = 404;
return(HttpContext.Response.WriteAsync("Database " + name + " wasn't found"));
}
UnprotectSecuredSettingsOfDatabaseDocument(dbDoc);
HttpContext.Response.StatusCode = 200;
using (var writer = new BlittableJsonTextWriter(context, ResponseBodyStream()))
{
writer.WriteDocument(context, new Document
{
Etag = etag,
Data = dbDoc,
});
writer.Flush();
}
}
return(Task.CompletedTask);
}
public async Task ReadDocWithCompressedStringFromOneContextAndWriteToAnother()
{
using (var documentStore = this.GetDocumentStore())
{
Server.ServerStore.Observer.Suspended = true;
var originalDoc = new Doc
{
Id = "doc/1",
StrVal = new string(Enumerable.Repeat('.', 129).ToArray()),
LongByteArray = Enumerable.Repeat((byte)2, 1024).ToArray()
};
using (var session = documentStore.OpenAsyncSession())
{
await session.StoreAsync(originalDoc);
await session.SaveChangesAsync();
}
var database = await Server.ServerStore.DatabasesLandlord.TryGetOrCreateResourceStore(documentStore.Database);
using (database.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext context))
using (context.OpenReadTransaction())
{
var doc = database.DocumentsStorage.Get(context, "doc/1");
MemoryStream ms = new MemoryStream();
using (var newContext = JsonOperationContext.ShortTermSingleUse())
using (var writer = new BlittableJsonTextWriter(newContext, ms))
{
writer.WriteDocument(newContext, doc, metadataOnly: false);
writer.Flush();
var bjro = GetReaderFromMemoryStream(ms, context);
var desereializedDoc = (Doc)EntityToBlittable.ConvertToEntity(typeof(Doc), null, bjro, DocumentConventions.Default);
Assert.Equal(originalDoc.StrVal, desereializedDoc.StrVal);
Assert.Equal(originalDoc.LongByteArray, originalDoc.LongByteArray);
}
}
}
}
public async Task Try()
{
using (ContextPool.AllocateOperationContext(out DocumentsOperationContext context))
{
var json = await context.ReadForMemoryAsync(RequestBodyStream(), null);
var tryout = JsonDeserializationServer.SubscriptionTryout(json);
var(collection, (script, functions), revisions) = SubscriptionConnection.ParseSubscriptionQuery(tryout.Query);
SubscriptionPatchDocument patch = null;
if (string.IsNullOrEmpty(script) == false)
{
patch = new SubscriptionPatchDocument(script, functions);
}
if (collection == null)
{
throw new ArgumentException("Collection must be specified");
}
var pageSize = GetIntValueQueryString("pageSize") ?? 1;
var state = new SubscriptionState
{
ChangeVectorForNextBatchStartingPoint = tryout.ChangeVector,
Query = tryout.Query
};
var fetcher = new SubscriptionDocumentsFetcher(Database, pageSize, -0x42,
new IPEndPoint(HttpContext.Connection.RemoteIpAddress, HttpContext.Connection.RemotePort), collection, revisions, state, patch);
if (Enum.TryParse(
tryout.ChangeVector,
out Constants.Documents.SubscriptionChangeVectorSpecialStates changeVectorSpecialValue))
{
switch (changeVectorSpecialValue)
{
case Constants.Documents.SubscriptionChangeVectorSpecialStates.BeginningOfTime:
case Constants.Documents.SubscriptionChangeVectorSpecialStates.DoNotChange:
state.ChangeVectorForNextBatchStartingPoint = null;
break;
case Constants.Documents.SubscriptionChangeVectorSpecialStates.LastDocument:
state.ChangeVectorForNextBatchStartingPoint = Database.DocumentsStorage.GetLastDocumentChangeVector(context, collection);
break;
}
}
using (var writer = new BlittableJsonTextWriter(context, ResponseBodyStream()))
{
writer.WriteStartObject();
writer.WritePropertyName("Results");
writer.WriteStartArray();
using (context.OpenReadTransaction())
{
var first = true;
foreach (var itemDetails in fetcher.GetDataToSend(context, 0))
{
if (itemDetails.Doc.Data == null)
{
continue;
}
if (first == false)
{
writer.WriteComma();
}
if (itemDetails.Exception == null)
{
writer.WriteDocument(context, itemDetails.Doc, metadataOnly: false);
}
else
{
var docWithExcepton = new DocumentWithException
{
Exception = itemDetails.Exception.ToString(),
ChangeVector = itemDetails.Doc.ChangeVector,
Id = itemDetails.Doc.Id,
DocumentData = itemDetails.Doc.Data
};
writer.WriteObject(context.ReadObject(docWithExcepton.ToJson(), ""));
}
first = false;
}
}
writer.WriteEndArray();
writer.WriteEndObject();
}
}
}
private async Task ProcessSubscriptionAsync()
{
if (_logger.IsInfoEnabled)
{
_logger.Info(
$"Starting processing documents for subscription {SubscriptionId} received from {TcpConnection.TcpClient.Client.RemoteEndPoint}");
}
using (DisposeOnDisconnect)
using (TcpConnection.DocumentDatabase.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext docsContext))
using (RegisterForNotificationOnNewDocuments())
{
var replyFromClientTask = GetReplyFromClientAsync();
string lastChangeVector = null;
string subscriptionChangeVectorBeforeCurrentBatch = SubscriptionState.ChangeVectorForNextBatchStartingPoint;
var startEtag = GetStartEtagForSubscription(docsContext, SubscriptionState);
var patch = SetupFilterScript();
var fetcher = new SubscriptionDocumentsFetcher(TcpConnection.DocumentDatabase, _options.MaxDocsPerBatch, SubscriptionId, TcpConnection.TcpClient.Client.RemoteEndPoint);
while (CancellationTokenSource.IsCancellationRequested == false)
{
bool anyDocumentsSentInCurrentIteration = false;
var sendingCurrentBatchStopwatch = Stopwatch.StartNew();
_buffer.SetLength(0);
var docsToFlush = 0;
using (TcpConnection.ContextPool.AllocateOperationContext(out JsonOperationContext context))
using (var writer = new BlittableJsonTextWriter(context, _buffer))
{
using (docsContext.OpenReadTransaction())
{
foreach (var result in fetcher.GetDataToSend(docsContext, Collection, Revisions, SubscriptionState, patch, startEtag))
{
startEtag = result.Doc.Etag;
lastChangeVector = string.IsNullOrEmpty(SubscriptionState.ChangeVectorForNextBatchStartingPoint)
? result.Doc.ChangeVector
: ChangeVectorUtils.MergeVectors(result.Doc.ChangeVector, SubscriptionState.ChangeVectorForNextBatchStartingPoint);
if (result.Doc.Data == null)
{
if (sendingCurrentBatchStopwatch.ElapsedMilliseconds > 1000)
{
await SendHeartBeat();
sendingCurrentBatchStopwatch.Restart();
}
continue;
}
anyDocumentsSentInCurrentIteration = true;
writer.WriteStartObject();
writer.WritePropertyName(context.GetLazyStringForFieldWithCaching(TypeSegment));
writer.WriteValue(BlittableJsonToken.String, context.GetLazyStringForFieldWithCaching(DataSegment));
writer.WriteComma();
writer.WritePropertyName(context.GetLazyStringForFieldWithCaching(DataSegment));
result.Doc.EnsureMetadata();
if (result.Exception != null)
{
var metadata = result.Doc.Data[Client.Constants.Documents.Metadata.Key];
writer.WriteValue(BlittableJsonToken.StartObject,
docsContext.ReadObject(new DynamicJsonValue
{
[Client.Constants.Documents.Metadata.Key] = metadata
}, result.Doc.Id)
);
writer.WriteComma();
writer.WritePropertyName(context.GetLazyStringForFieldWithCaching(ExceptionSegment));
writer.WriteValue(BlittableJsonToken.String, context.GetLazyStringForFieldWithCaching(result.Exception.ToString()));
}
else
{
writer.WriteDocument(docsContext, result.Doc, metadataOnly: false);
}
writer.WriteEndObject();
docsToFlush++;
// perform flush for current batch after 1000ms of running or 1 MB
if (_buffer.Length > Constants.Size.Megabyte ||
sendingCurrentBatchStopwatch.ElapsedMilliseconds > 1000)
{
if (docsToFlush > 0)
{
await FlushDocsToClient(writer, docsToFlush);
docsToFlush = 0;
sendingCurrentBatchStopwatch.Restart();
}
else
{
await SendHeartBeat();
}
}
}
}
if (anyDocumentsSentInCurrentIteration)
{
context.Write(writer, new DynamicJsonValue
{
[nameof(SubscriptionConnectionServerMessage.Type)] = nameof(SubscriptionConnectionServerMessage.MessageType.EndOfBatch)
});
await FlushDocsToClient(writer, docsToFlush, true);
if (_logger.IsInfoEnabled)
{
_logger.Info(
$"Finished sending a batch with {docsToFlush} documents for subscription {Options.SubscriptionName}");
}
}
}
if (anyDocumentsSentInCurrentIteration == false)
{
if (_logger.IsInfoEnabled)
{
_logger.Info(
$"Finished sending a batch with {docsToFlush} documents for subscription {Options.SubscriptionName}");
}
await TcpConnection.DocumentDatabase.SubscriptionStorage.AcknowledgeBatchProcessed(SubscriptionId,
Options.SubscriptionName,
lastChangeVector,
subscriptionChangeVectorBeforeCurrentBatch);
subscriptionChangeVectorBeforeCurrentBatch = lastChangeVector;
if (sendingCurrentBatchStopwatch.ElapsedMilliseconds > 1000)
{
await SendHeartBeat();
}
using (docsContext.OpenReadTransaction())
{
long globalEtag = TcpConnection.DocumentDatabase.DocumentsStorage.GetLastDocumentEtag(docsContext, Collection);
if (globalEtag > startEtag)
{
continue;
}
}
if (await WaitForChangedDocuments(replyFromClientTask))
{
continue;
}
}
SubscriptionConnectionClientMessage clientReply;
while (true)
{
var result = await Task.WhenAny(replyFromClientTask,
TimeoutManager.WaitFor(TimeSpan.FromMilliseconds(5000), CancellationTokenSource.Token)).ConfigureAwait(false);
CancellationTokenSource.Token.ThrowIfCancellationRequested();
if (result == replyFromClientTask)
{
clientReply = await replyFromClientTask;
if (clientReply.Type == SubscriptionConnectionClientMessage.MessageType.DisposedNotification)
{
CancellationTokenSource.Cancel();
break;
}
replyFromClientTask = GetReplyFromClientAsync();
break;
}
await SendHeartBeat();
}
CancellationTokenSource.Token.ThrowIfCancellationRequested();
switch (clientReply.Type)
{
case SubscriptionConnectionClientMessage.MessageType.Acknowledge:
await TcpConnection.DocumentDatabase.SubscriptionStorage.AcknowledgeBatchProcessed(
SubscriptionId,
Options.SubscriptionName,
lastChangeVector,
subscriptionChangeVectorBeforeCurrentBatch);
subscriptionChangeVectorBeforeCurrentBatch = lastChangeVector;
Stats.LastAckReceivedAt = DateTime.UtcNow;
Stats.AckRate.Mark();
await WriteJsonAsync(new DynamicJsonValue
{
[nameof(SubscriptionConnectionServerMessage.Type)] = nameof(SubscriptionConnectionServerMessage.MessageType.Confirm)
});
break;
//precaution, should not reach this case...
case SubscriptionConnectionClientMessage.MessageType.DisposedNotification:
CancellationTokenSource.Cancel();
break;
default:
throw new ArgumentException("Unknown message type from client " +
clientReply.Type);
}
}
CancellationTokenSource.Token.ThrowIfCancellationRequested();
}
}
public async Task Try()
{
using (ContextPool.AllocateOperationContext(out DocumentsOperationContext context))
{
BlittableJsonReaderObject json = await context.ReadForMemoryAsync(RequestBodyStream(), null);
SubscriptionTryout tryout = JsonDeserializationServer.SubscriptionTryout(json);
SubscriptionConnection.ParsedSubscription sub = SubscriptionConnection.ParseSubscriptionQuery(tryout.Query);
SubscriptionPatchDocument patch = null;
if (string.IsNullOrEmpty(sub.Script) == false)
{
patch = new SubscriptionPatchDocument(sub.Script, sub.Functions);
}
if (sub.Collection == null)
{
throw new ArgumentException("Collection must be specified");
}
const int maxPageSize = 1024;
var pageSize = GetIntValueQueryString("pageSize") ?? 1;
if (pageSize > maxPageSize)
{
throw new ArgumentException($"Cannot gather more than {maxPageSize} results during tryouts, but requested number was {pageSize}.");
}
var state = new SubscriptionState
{
ChangeVectorForNextBatchStartingPoint = tryout.ChangeVector,
Query = tryout.Query
};
var fetcher = new SubscriptionDocumentsFetcher(Database, int.MaxValue, -0x42,
new IPEndPoint(HttpContext.Connection.RemoteIpAddress, HttpContext.Connection.RemotePort), sub.Collection, sub.Revisions, state, patch);
var includeCmd = new IncludeDocumentsCommand(Database.DocumentsStorage, context, sub.Includes, isProjection: patch != null);
if (Enum.TryParse(
tryout.ChangeVector,
out Constants.Documents.SubscriptionChangeVectorSpecialStates changeVectorSpecialValue))
{
switch (changeVectorSpecialValue)
{
case Constants.Documents.SubscriptionChangeVectorSpecialStates.BeginningOfTime:
case Constants.Documents.SubscriptionChangeVectorSpecialStates.DoNotChange:
state.ChangeVectorForNextBatchStartingPoint = null;
break;
case Constants.Documents.SubscriptionChangeVectorSpecialStates.LastDocument:
using (context.OpenReadTransaction())
{
state.ChangeVectorForNextBatchStartingPoint = Database.DocumentsStorage.GetLastDocumentChangeVector(context.Transaction.InnerTransaction, context, sub.Collection);
}
break;
}
}
else
{
state.ChangeVectorForNextBatchStartingPoint = tryout.ChangeVector;
}
var changeVector = state.ChangeVectorForNextBatchStartingPoint.ToChangeVector();
var cv = changeVector.FirstOrDefault(x => x.DbId == Database.DbBase64Id);
var sp = Stopwatch.StartNew();
var timeLimit = TimeSpan.FromSeconds(GetIntValueQueryString("timeLimit", false) ?? 15);
var startEtag = cv.Etag;
using (var writer = new BlittableJsonTextWriter(context, ResponseBodyStream()))
using (context.OpenReadTransaction())
{
writer.WriteStartObject();
writer.WritePropertyName("Results");
writer.WriteStartArray();
var numberOfDocs = 0;
while (numberOfDocs == 0 && sp.Elapsed < timeLimit)
{
var first = true;
var lastEtag = startEtag;
foreach (var itemDetails in fetcher.GetDataToSend(context, includeCmd, startEtag))
{
if (itemDetails.Doc.Data != null)
{
using (itemDetails.Doc.Data)
{
includeCmd.Gather(itemDetails.Doc);
if (first == false)
{
writer.WriteComma();
}
if (itemDetails.Exception == null)
{
writer.WriteDocument(context, itemDetails.Doc, metadataOnly: false);
}
else
{
var documentWithException = new DocumentWithException
{
Exception = itemDetails.Exception.ToString(),
ChangeVector = itemDetails.Doc.ChangeVector,
Id = itemDetails.Doc.Id,
DocumentData = itemDetails.Doc.Data
};
writer.WriteObject(context.ReadObject(documentWithException.ToJson(), ""));
}
first = false;
if (++numberOfDocs >= pageSize)
{
break;
}
}
}
if (sp.Elapsed >= timeLimit)
{
break;
}
lastEtag = itemDetails.Doc.Etag;
}
if (startEtag == lastEtag)
{
break;
}
startEtag = lastEtag;
}
writer.WriteEndArray();
writer.WriteComma();
writer.WritePropertyName("Includes");
var includes = new List <Document>();
includeCmd.Fill(includes);
writer.WriteIncludes(context, includes);
writer.WriteEndObject();
}
}
}
public async Task SubscriptionShouldRespectDocumentsWithCompressedData()
{
using (var documentStore = this.GetDocumentStore())
{
Server.ServerStore.Observer.Suspended = true;
var originalDoc = new Doc
{
Id = "doc/1",
StrVal = new string(Enumerable.Repeat('.', 129).ToArray()),
LongByteArray = Enumerable.Repeat((byte)2, 1024).ToArray()
};
using (var session = documentStore.OpenAsyncSession())
{
await session.StoreAsync(originalDoc);
await session.SaveChangesAsync();
}
var database = await Server.ServerStore.DatabasesLandlord.TryGetOrCreateResourceStore(documentStore.Database);
using (database.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext context))
using (context.OpenReadTransaction())
{
var doc = database.DocumentsStorage.Get(context, "doc/1");
MemoryStream ms = new MemoryStream();
using (var newContext = JsonOperationContext.ShortTermSingleUse())
using (var writer = new BlittableJsonTextWriter(newContext, ms))
{
writer.WriteDocument(newContext, doc, metadataOnly: false);
writer.Flush();
var bjro = GetReaderFromMemoryStream(ms, context);
var desereializedDoc = (Doc)EntityToBlittable.ConvertToEntity(typeof(Doc), null, bjro, DocumentConventions.Default);
Assert.Equal(originalDoc.StrVal, desereializedDoc.StrVal);
Assert.Equal(originalDoc.LongByteArray, originalDoc.LongByteArray);
}
}
var subscriptionCreationParams = new SubscriptionCreationOptions
{
Query = "from Docs",
};
var subsId = await documentStore.Subscriptions.CreateAsync(subscriptionCreationParams).ConfigureAwait(false);
var amre = new AsyncManualResetEvent();
using (var subscription = documentStore.Subscriptions.GetSubscriptionWorker <Doc>(new SubscriptionWorkerOptions(subsId)))
{
var t = subscription.Run(batch =>
{
var receivedDoc = batch.Items.First().Result;
Assert.Equal(originalDoc.LongByteArray, receivedDoc.LongByteArray);
Assert.Equal(originalDoc.StrVal, receivedDoc.StrVal);
amre.Set();
});
try
{
Assert.True(await amre.WaitAsync(TimeSpan.FromSeconds(60)));
}
catch
{
if (t.IsFaulted)
{
t.Wait();
}
throw;
}
}
}
}
/// <summary>
/// Iterates on a batch in document collection, process it and send documents if found any match
/// </summary>
/// <param name="docsContext"></param>
/// <param name="sendingCurrentBatchStopwatch"></param>
/// <returns>Whether succeeded finding any documents to send</returns>
private async Task <bool> TrySendingBatchToClient(DocumentsOperationContext docsContext, Stopwatch sendingCurrentBatchStopwatch)
{
bool anyDocumentsSentInCurrentIteration = false;
int docsToFlush = 0;
using (var writer = new BlittableJsonTextWriter(docsContext, _buffer))
{
using (docsContext.OpenReadTransaction())
{
foreach (var result in _documentsFetcher.GetDataToSend(docsContext, _startEtag))
{
_startEtag = result.Doc.Etag;
_lastChangeVector = string.IsNullOrEmpty(SubscriptionState.ChangeVectorForNextBatchStartingPoint)
? result.Doc.ChangeVector
: ChangeVectorUtils.MergeVectors(result.Doc.ChangeVector, SubscriptionState.ChangeVectorForNextBatchStartingPoint);
if (result.Doc.Data == null)
{
if (sendingCurrentBatchStopwatch.ElapsedMilliseconds > 1000)
{
await SendHeartBeat();
sendingCurrentBatchStopwatch.Restart();
}
continue;
}
anyDocumentsSentInCurrentIteration = true;
writer.WriteStartObject();
writer.WritePropertyName(docsContext.GetLazyStringForFieldWithCaching(TypeSegment));
writer.WriteValue(BlittableJsonToken.String, docsContext.GetLazyStringForFieldWithCaching(DataSegment));
writer.WriteComma();
writer.WritePropertyName(docsContext.GetLazyStringForFieldWithCaching(DataSegment));
result.Doc.EnsureMetadata();
if (result.Exception != null)
{
var metadata = result.Doc.Data[Client.Constants.Documents.Metadata.Key];
writer.WriteValue(BlittableJsonToken.StartObject,
docsContext.ReadObject(new DynamicJsonValue
{
[Client.Constants.Documents.Metadata.Key] = metadata
}, result.Doc.Id)
);
writer.WriteComma();
writer.WritePropertyName(docsContext.GetLazyStringForFieldWithCaching(ExceptionSegment));
writer.WriteValue(BlittableJsonToken.String, docsContext.GetLazyStringForFieldWithCaching(result.Exception.ToString()));
}
else
{
writer.WriteDocument(docsContext, result.Doc, metadataOnly: false);
}
writer.WriteEndObject();
docsToFlush++;
// perform flush for current batch after 1000ms of running or 1 MB
if (_buffer.Length > Constants.Size.Megabyte ||
sendingCurrentBatchStopwatch.ElapsedMilliseconds > 1000)
{
if (docsToFlush > 0)
{
await FlushDocsToClient(writer, docsToFlush);
docsToFlush = 0;
sendingCurrentBatchStopwatch.Restart();
}
else
{
await SendHeartBeat();
}
}
}
}
if (anyDocumentsSentInCurrentIteration)
{
docsContext.Write(writer, new DynamicJsonValue
{
[nameof(SubscriptionConnectionServerMessage.Type)] = nameof(SubscriptionConnectionServerMessage.MessageType.EndOfBatch)
});
await FlushDocsToClient(writer, docsToFlush, true);
if (_logger.IsInfoEnabled)
{
_logger.Info(
$"Finished sending a batch with {docsToFlush} documents for subscription {Options.SubscriptionName}");
}
}
}
return(anyDocumentsSentInCurrentIteration);
}
