public IEnumerable <(Document Doc, Exception Exception)> GetDataToSend(
DocumentsOperationContext docsContext,
string collection,
bool revisions,
SubscriptionState subscription,
SubscriptionPatchDocument patch,
long startEtag)
{
if (string.IsNullOrEmpty(collection))
{
throw new ArgumentException("The collection name must be specified");
}
if (revisions)
{
if (_db.DocumentsStorage.RevisionsStorage.Configuration == null ||
_db.DocumentsStorage.RevisionsStorage.GetRevisionsConfiguration(collection).Active == false)
{
throw new SubscriptionInvalidStateException($"Cannot use a revisions subscription, database {_db.Name} does not have revisions configuration.");
}
return(GetRevisionsToSend(docsContext, collection, subscription, startEtag, patch));
}
return(GetDocumentsToSend(docsContext, collection, subscription, startEtag, patch));
}
private SubscriptionPatchDocument SetupFilterScript()
{
SubscriptionPatchDocument patch = null;
if (string.IsNullOrWhiteSpace(Script) == false)
{
patch = new SubscriptionPatchDocument(Script, Functions);
}
return(patch);
}
private SubscriptionPatchDocument SetupFilterScript(SubscriptionCriteria criteria)
{
SubscriptionPatchDocument patch = null;
if (string.IsNullOrWhiteSpace(criteria.FilterJavaScript) == false)
{
patch = new SubscriptionPatchDocument(TcpConnection.DocumentDatabase, criteria.FilterJavaScript);
}
return(patch);
}
public SubscriptionDocumentsFetcher(DocumentDatabase db, int maxBatchSize, long subscriptionId, EndPoint remoteEndpoint, string collection,
bool revisions,
SubscriptionState subscription,
SubscriptionPatchDocument patch)
{
_db = db;
_logger = LoggingSource.Instance.GetLogger <SubscriptionDocumentsFetcher>(db.Name);
_maxBatchSize = maxBatchSize;
_subscriptionId = subscriptionId;
_remoteEndpoint = remoteEndpoint;
_collection = collection;
_revisions = revisions;
_subscription = subscription;
_patch = patch;
}
public SubscriptionDocumentsFetcher(DocumentDatabase db, int maxBatchSize, long subscriptionId, EndPoint remoteEndpoint, string collection,
bool revisions,
SubscriptionState subscription,
SubscriptionPatchDocument patch)
{
_db = db;
_logger = LoggingSource.Instance.GetLogger <SubscriptionDocumentsFetcher>(db.Name);
_maxBatchSize = maxBatchSize;
_subscriptionId = subscriptionId;
_remoteEndpoint = remoteEndpoint;
_collection = collection;
_revisions = revisions;
_subscription = subscription;
_patch = patch;
_maximumAllowedMemory = new Size((_db.Is32Bits ? 4 : 32) * Voron.Global.Constants.Size.Megabyte, SizeUnit.Bytes);
}
public SubscriptionDocumentsFetcher(DocumentDatabase db, int maxBatchSize, long subscriptionId, EndPoint remoteEndpoint, string collection,
bool revisions,
SubscriptionState subscription,
SubscriptionPatchDocument patch)
{
_db = db;
_logger = LoggingSource.Instance.GetLogger <SubscriptionDocumentsFetcher>(db.Name);
_maxBatchSize = maxBatchSize;
_subscriptionId = subscriptionId;
_remoteEndpoint = remoteEndpoint;
_collection = collection;
_revisions = revisions;
_subscription = subscription;
_patch = patch;
_maximumAllowedMemory = new Size((PlatformDetails.Is32Bits ||
_db.Configuration.Storage.ForceUsing32BitsPager) ? 4 : 32, SizeUnit.Megabytes);
}
private bool DocumentMatchCriteriaScript(SubscriptionPatchDocument patch, DocumentsOperationContext dbContext,
Document doc)
{
if (patch == null)
{
return(true);
}
try
{
return(patch.MatchCriteria(dbContext, doc));
}
catch (Exception ex)
{
if (_logger.IsInfoEnabled)
{
_logger.Info(
$"Criteria script threw exception for subscription {_options.SubscriptionId} connected to {TcpConnection.TcpClient.Client.RemoteEndPoint} for document id {doc.Key}",
ex);
}
return(false);
}
}
public async Task Try()
{
using (ContextPool.AllocateOperationContext(out DocumentsOperationContext context))
{
var json = await context.ReadForMemoryAsync(RequestBodyStream(), null);
var tryout = JsonDeserializationServer.SubscriptionTryout(json);
var(collection, (script, functions), revisions) = SubscriptionConnection.ParseSubscriptionQuery(tryout.Query);
SubscriptionPatchDocument patch = null;
if (string.IsNullOrEmpty(script) == false)
{
patch = new SubscriptionPatchDocument(script, functions);
}
if (collection == null)
{
throw new ArgumentException("Collection must be specified");
}
var pageSize = GetIntValueQueryString("pageSize") ?? 1;
var state = new SubscriptionState
{
ChangeVectorForNextBatchStartingPoint = tryout.ChangeVector,
Query = tryout.Query
};
var fetcher = new SubscriptionDocumentsFetcher(Database, pageSize, -0x42,
new IPEndPoint(HttpContext.Connection.RemoteIpAddress, HttpContext.Connection.RemotePort), collection, revisions, state, patch);
if (Enum.TryParse(
tryout.ChangeVector,
out Constants.Documents.SubscriptionChangeVectorSpecialStates changeVectorSpecialValue))
{
switch (changeVectorSpecialValue)
{
case Constants.Documents.SubscriptionChangeVectorSpecialStates.BeginningOfTime:
case Constants.Documents.SubscriptionChangeVectorSpecialStates.DoNotChange:
state.ChangeVectorForNextBatchStartingPoint = null;
break;
case Constants.Documents.SubscriptionChangeVectorSpecialStates.LastDocument:
state.ChangeVectorForNextBatchStartingPoint = Database.DocumentsStorage.GetLastDocumentChangeVector(context, collection);
break;
}
}
using (var writer = new BlittableJsonTextWriter(context, ResponseBodyStream()))
{
writer.WriteStartObject();
writer.WritePropertyName("Results");
writer.WriteStartArray();
using (context.OpenReadTransaction())
{
var first = true;
foreach (var itemDetails in fetcher.GetDataToSend(context, 0))
{
if (itemDetails.Doc.Data == null)
{
continue;
}
if (first == false)
{
writer.WriteComma();
}
if (itemDetails.Exception == null)
{
writer.WriteDocument(context, itemDetails.Doc, metadataOnly: false);
}
else
{
var docWithExcepton = new DocumentWithException
{
Exception = itemDetails.Exception.ToString(),
ChangeVector = itemDetails.Doc.ChangeVector,
Id = itemDetails.Doc.Id,
DocumentData = itemDetails.Doc.Data
};
writer.WriteObject(context.ReadObject(docWithExcepton.ToJson(), ""));
}
first = false;
}
}
writer.WriteEndArray();
writer.WriteEndObject();
}
}
}
private IEnumerable <(Document Doc, Exception Exception)> GetDocumentsToSend(DocumentsOperationContext docsContext,
string collection,
SubscriptionState subscription,
long startEtag, SubscriptionPatchDocument patch)
{
int size = 0;
using (_db.Scripts.GetScriptRunner(patch, true, out var run))
{
foreach (var doc in _db.DocumentsStorage.GetDocumentsFrom(
docsContext,
collection,
startEtag + 1,
0,
int.MaxValue))
{
using (doc.Data)
{
if (ShouldSendDocument(subscription, run, patch, docsContext, doc, out BlittableJsonReaderObject transformResult, out var exception) == false)
{
if (exception != null)
{
yield return(doc, exception);
if (++size >= _maxBatchSize)
{
yield break;
}
}
else
{
doc.Data = null;
yield return(doc, null);
}
doc.Data = null;
}
public async Task Try()
{
using (ContextPool.AllocateOperationContext(out DocumentsOperationContext context))
{
BlittableJsonReaderObject json = await context.ReadForMemoryAsync(RequestBodyStream(), null);
SubscriptionTryout tryout = JsonDeserializationServer.SubscriptionTryout(json);
SubscriptionConnection.ParsedSubscription sub = SubscriptionConnection.ParseSubscriptionQuery(tryout.Query);
SubscriptionPatchDocument patch = null;
if (string.IsNullOrEmpty(sub.Script) == false)
{
patch = new SubscriptionPatchDocument(sub.Script, sub.Functions);
}
if (sub.Collection == null)
{
throw new ArgumentException("Collection must be specified");
}
const int maxPageSize = 1024;
var pageSize = GetIntValueQueryString("pageSize") ?? 1;
if (pageSize > maxPageSize)
{
throw new ArgumentException($"Cannot gather more than {maxPageSize} results during tryouts, but requested number was {pageSize}.");
}
var state = new SubscriptionState
{
ChangeVectorForNextBatchStartingPoint = tryout.ChangeVector,
Query = tryout.Query
};
var fetcher = new SubscriptionDocumentsFetcher(Database, int.MaxValue, -0x42,
new IPEndPoint(HttpContext.Connection.RemoteIpAddress, HttpContext.Connection.RemotePort), sub.Collection, sub.Revisions, state, patch);
var includeCmd = new IncludeDocumentsCommand(Database.DocumentsStorage, context, sub.Includes, isProjection: patch != null);
if (Enum.TryParse(
tryout.ChangeVector,
out Constants.Documents.SubscriptionChangeVectorSpecialStates changeVectorSpecialValue))
{
switch (changeVectorSpecialValue)
{
case Constants.Documents.SubscriptionChangeVectorSpecialStates.BeginningOfTime:
case Constants.Documents.SubscriptionChangeVectorSpecialStates.DoNotChange:
state.ChangeVectorForNextBatchStartingPoint = null;
break;
case Constants.Documents.SubscriptionChangeVectorSpecialStates.LastDocument:
using (context.OpenReadTransaction())
{
state.ChangeVectorForNextBatchStartingPoint = Database.DocumentsStorage.GetLastDocumentChangeVector(context.Transaction.InnerTransaction, context, sub.Collection);
}
break;
}
}
else
{
state.ChangeVectorForNextBatchStartingPoint = tryout.ChangeVector;
}
var changeVector = state.ChangeVectorForNextBatchStartingPoint.ToChangeVector();
var cv = changeVector.FirstOrDefault(x => x.DbId == Database.DbBase64Id);
var sp = Stopwatch.StartNew();
var timeLimit = TimeSpan.FromSeconds(GetIntValueQueryString("timeLimit", false) ?? 15);
var startEtag = cv.Etag;
await using (var writer = new AsyncBlittableJsonTextWriter(context, ResponseBodyStream()))
using (context.OpenReadTransaction())
{
writer.WriteStartObject();
writer.WritePropertyName("Results");
writer.WriteStartArray();
var numberOfDocs = 0;
while (numberOfDocs == 0 && sp.Elapsed < timeLimit)
{
var first = true;
var lastEtag = startEtag;
foreach (var itemDetails in fetcher.GetDataToSend(context, includeCmd, startEtag))
{
if (itemDetails.Doc.Data != null)
{
using (itemDetails.Doc.Data)
{
includeCmd.Gather(itemDetails.Doc);
if (first == false)
{
writer.WriteComma();
}
if (itemDetails.Exception == null)
{
writer.WriteDocument(context, itemDetails.Doc, metadataOnly: false);
}
else
{
var documentWithException = new DocumentWithException
{
Exception = itemDetails.Exception.ToString(),
ChangeVector = itemDetails.Doc.ChangeVector,
Id = itemDetails.Doc.Id,
DocumentData = itemDetails.Doc.Data
};
writer.WriteObject(context.ReadObject(documentWithException.ToJson(), ""));
}
first = false;
if (++numberOfDocs >= pageSize)
{
break;
}
}
}
if (sp.Elapsed >= timeLimit)
{
break;
}
lastEtag = itemDetails.Doc.Etag;
}
if (startEtag == lastEtag)
{
break;
}
startEtag = lastEtag;
}
writer.WriteEndArray();
writer.WriteComma();
writer.WritePropertyName("Includes");
var includes = new List <Document>();
includeCmd.Fill(includes);
await writer.WriteIncludesAsync(context, includes);
writer.WriteEndObject();
}
}
}
private async Task ProcessSubscriptionAsync()
{
if (_logger.IsInfoEnabled)
{
_logger.Info(
$"Starting processing documents for subscription {SubscriptionId} received from {TcpConnection.TcpClient.Client.RemoteEndPoint}");
}
using (DisposeOnDisconnect)
using (RegisterForNotificationOnNewDocuments())
{
var replyFromClientTask = GetReplyFromClientAsync();
string subscriptionChangeVectorBeforeCurrentBatch = SubscriptionState.ChangeVectorForNextBatchStartingPoint;
_startEtag = GetStartEtagForSubscription(SubscriptionState);
_filterAndProjectionScript = SetupFilterAndProjectionScript();
_documentsFetcher = new SubscriptionDocumentsFetcher(TcpConnection.DocumentDatabase, _options.MaxDocsPerBatch, SubscriptionId, TcpConnection.TcpClient.Client.RemoteEndPoint, Collection, Revisions, SubscriptionState, _filterAndProjectionScript);
while (CancellationTokenSource.IsCancellationRequested == false)
{
_buffer.SetLength(0);
using (TcpConnection.DocumentDatabase.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext docsContext))
{
var sendingCurrentBatchStopwatch = Stopwatch.StartNew();
var anyDocumentsSentInCurrentIteration = await TrySendingBatchToClient(docsContext, sendingCurrentBatchStopwatch);
if (anyDocumentsSentInCurrentIteration == false)
{
if (_logger.IsInfoEnabled)
{
_logger.Info(
$"Did not find any documents to send for subscription {Options.SubscriptionName}");
}
await TcpConnection.DocumentDatabase.SubscriptionStorage.AcknowledgeBatchProcessed(SubscriptionId,
Options.SubscriptionName,
_lastChangeVector,
subscriptionChangeVectorBeforeCurrentBatch);
subscriptionChangeVectorBeforeCurrentBatch = _lastChangeVector;
if (sendingCurrentBatchStopwatch.ElapsedMilliseconds > 1000)
{
await SendHeartBeat();
}
using (docsContext.OpenReadTransaction())
{
long globalEtag = TcpConnection.DocumentDatabase.DocumentsStorage.GetLastDocumentEtag(docsContext, Collection);
if (globalEtag > _startEtag)
{
continue;
}
}
if (await WaitForChangedDocuments(replyFromClientTask))
{
continue;
}
}
}
(replyFromClientTask, subscriptionChangeVectorBeforeCurrentBatch) =
await WaitForClientAck(replyFromClientTask, subscriptionChangeVectorBeforeCurrentBatch);
}
CancellationTokenSource.Token.ThrowIfCancellationRequested();
}
}
