public Task <FacetedQueryResult> ExecuteFacetedQuery(string indexName, FacetQuery query, long?facetsEtag, long?existingResultEtag, OperationCancelToken token)
{
if (query.FacetSetupDoc != null)
{
FacetSetup facetSetup;
using (_documentsContext.OpenReadTransaction())
{
var facetSetupAsJson = _database.DocumentsStorage.Get(_documentsContext, query.FacetSetupDoc);
if (facetSetupAsJson == null)
{
throw new DocumentDoesNotExistException(query.FacetSetupDoc);
}
try
{
facetSetup = JsonDeserializationServer.FacetSetup(facetSetupAsJson.Data);
}
catch (Exception e)
{
throw new DocumentParseException(query.FacetSetupDoc, typeof(FacetSetup), e);
}
facetsEtag = facetSetupAsJson.Etag;
}
query.Facets = facetSetup.Facets;
}
return(ExecuteFacetedQuery(indexName, query, facetsEtag.Value, existingResultEtag, token));
}
private async Task ApplyRetention(
DocumentsOperationContext context,
TimeSeriesCollectionConfiguration config,
CollectionName collectionName,
TimeSeriesPolicy policy,
DateTime now)
{
var tss = context.DocumentDatabase.DocumentsStorage.TimeSeriesStorage;
if (policy.RetentionTime == TimeValue.MaxValue)
{
return;
}
var to = now.Add(-policy.RetentionTime);
var list = new List <Slice>();
while (true)
{
Cts.Token.ThrowIfCancellationRequested();
context.Reset();
context.Renew();
list.Clear();
using (context.OpenReadTransaction())
{
foreach (var item in tss.Stats.GetTimeSeriesByPolicyFromStartDate(context, collectionName, policy.Name, to, TimeSeriesRollups.TimeSeriesRetentionCommand.BatchSize))
{
if (RequiredForNextPolicy(context, config, policy, item, to))
{
continue;
}
if (tss.Rollups.HasPendingRollupFrom(context, item, to) == false)
{
list.Add(item);
}
}
if (list.Count == 0)
{
return;
}
if (Logger.IsInfoEnabled)
{
Logger.Info($"Found {list.Count} time-series for retention in policy {policy.Name} with collection '{collectionName.Name}' up-to {to}"
#if DEBUG
+ $"{Environment.NewLine}{string.Join(Environment.NewLine, list)}"
#endif
);
}
var cmd = new TimeSeriesRollups.TimeSeriesRetentionCommand(list, collectionName.Name, to);
await _database.TxMerger.Enqueue(cmd);
}
}
}
public MoreLikeThisQueryResultServerSide ExecuteMoreLikeThisQuery(string indexName, MoreLikeThisQueryServerSide query, DocumentsOperationContext context, long?existingResultEtag, OperationCancelToken token)
{
if (query == null)
{
throw new ArgumentNullException(nameof(query));
}
if (string.IsNullOrEmpty(query.DocumentId) && query.MapGroupFields.Count == 0)
{
throw new InvalidOperationException("The document id or map group fields are mandatory");
}
var index = GetIndex(indexName);
var etag = index.GetIndexEtag();
if (etag == existingResultEtag)
{
return(MoreLikeThisQueryResultServerSide.NotModifiedResult);
}
context.OpenReadTransaction();
return(index.MoreLikeThisQuery(query, context, token));
}
public MoreLikeThisQueryResultServerSide ExecuteMoreLikeThisQuery(MoreLikeThisQueryServerSide query, DocumentsOperationContext context, long?existingResultEtag, OperationCancelToken token)
{
if (query == null)
{
throw new ArgumentNullException(nameof(query));
}
if (string.IsNullOrWhiteSpace(query.DocumentId) &&
query.MapGroupFields.Count == 0 &&
string.IsNullOrWhiteSpace(query.Document))
{
throw new InvalidOperationException("The document id, document (artificial document property) or map group fields are mandatory");
}
var sw = Stopwatch.StartNew();
var index = GetIndex(query.Metadata.IndexName);
if (existingResultEtag.HasValue)
{
var etag = index.GetIndexEtag();
if (etag == existingResultEtag.Value)
{
return(MoreLikeThisQueryResultServerSide.NotModifiedResult);
}
}
context.OpenReadTransaction();
var result = index.MoreLikeThisQuery(query, context, token);
result.DurationInMs = (int)sw.Elapsed.TotalMilliseconds;
return(result);
}
public IDisposable Initialize(DatabaseSmugglerOptions options, SmugglerResult result, out long buildVersion)
{
_currentTypeIndex = 0;
if (options.OperateOnTypes.HasFlag(DatabaseItemType.Documents) ||
options.OperateOnTypes.HasFlag(DatabaseItemType.RevisionDocuments) ||
options.OperateOnTypes.HasFlag(DatabaseItemType.Tombstones) ||
options.OperateOnTypes.HasFlag(DatabaseItemType.Conflicts) ||
options.OperateOnTypes.HasFlag(DatabaseItemType.Counters))
{
_returnContext = _database.DocumentsStorage.ContextPool.AllocateOperationContext(out _context);
_disposeTransaction = _context.OpenReadTransaction();
LastEtag = DocumentsStorage.ReadLastEtag(_disposeTransaction.InnerTransaction);
}
if (options.OperateOnTypes.HasFlag(DatabaseItemType.CompareExchange) ||
options.OperateOnTypes.HasFlag(DatabaseItemType.Identities))
{
_returnServerContext = _database.ServerStore.ContextPool.AllocateOperationContext(out _serverContext);
_disposeServerTransaction = _serverContext.OpenReadTransaction();
}
buildVersion = ServerVersion.Build;
return(new DisposableAction(() =>
{
_disposeServerTransaction?.Dispose();
_returnServerContext?.Dispose();
_disposeTransaction?.Dispose();
_returnContext?.Dispose();
}));
}
public override Task <DocumentQueryResult> ExecuteQuery(IndexQueryServerSide query, DocumentsOperationContext documentsContext, long?existingResultEtag, OperationCancelToken token)
{
var result = new DocumentQueryResult();
if (documentsContext.Transaction == null || documentsContext.Transaction.Disposed)
{
documentsContext.OpenReadTransaction();
}
FillCountOfResultsAndIndexEtag(result, query.Metadata, documentsContext);
if (query.Metadata.HasOrderByRandom == false && existingResultEtag.HasValue)
{
if (result.ResultEtag == existingResultEtag)
{
return(Task.FromResult(DocumentQueryResult.NotModifiedResult));
}
}
var collection = GetCollectionName(query.Metadata.CollectionName, out var indexName);
using (QueryRunner.MarkQueryAsRunning(indexName, query, token))
{
result.IndexName = indexName;
ExecuteCollectionQuery(result, query, collection, documentsContext, pulseReadingTransaction: false, token.Token);
return(Task.FromResult(result));
}
}
public async Task CreateAutoIndexesAndWaitIfNecessary()
{
var queryStepsGatherer = new QueryQueryStepGatherer();
queryStepsGatherer.Visit(RootQueryStep);
if (_context.Transaction == null || _context.Transaction.Disposed)
{
_context.OpenReadTransaction();
}
try
{
var etag = DocumentsStorage.ReadLastEtag(_context.Transaction.InnerTransaction);
var queryDuration = Stopwatch.StartNew();
var indexes = new List <Index>();
var indexWaiters = new Dictionary <Index, (IndexQueryServerSide, AsyncWaitForIndexing)>();
foreach (var queryStepInfo in queryStepsGatherer.QuerySteps)
{
if (string.IsNullOrWhiteSpace(queryStepInfo.QueryStep.Query.From.From.FieldValue) || queryStepInfo.IsIndexQuery)
{
continue;
}
var indexQuery = new IndexQueryServerSide(queryStepInfo.QueryStep.GetQueryString, queryStepInfo.QueryStep.QueryParameters);
//No sense creating an index for collection queries
if (indexQuery.Metadata.IsCollectionQuery)
{
continue;
}
var indexCreationInfo = await _dynamicQueryRunner.CreateAutoIndexIfNeeded(indexQuery, true, null, _database.DatabaseShutdown);
if (indexCreationInfo.HasCreatedAutoIndex) //wait for non-stale only IF we just created an auto-index
{
indexes.Add(indexCreationInfo.Index);
var queryTimeout = indexQuery.WaitForNonStaleResultsTimeout ?? Index.DefaultWaitForNonStaleResultsTimeout;
indexWaiters.Add(indexCreationInfo.Index, (indexQuery, new AsyncWaitForIndexing(queryDuration, queryTimeout, indexCreationInfo.Index)));
}
}
await WaitForNonStaleResultsInternal(etag, indexes, indexWaiters);
}
finally
{
//The rest of the code assumes that a Tx is not opened
_context.Transaction.Dispose();
}
}
public static FacetQuery Create(DocumentsOperationContext context, IndexQueryServerSide query)
{
long?facetsEtag = null;
DocumentsTransaction tx = null;
try
{
var facets = new Dictionary <string, FacetSetup>(StringComparer.OrdinalIgnoreCase);
foreach (var selectField in query.Metadata.SelectFields)
{
if (selectField.IsFacet == false)
{
continue;
}
var facetField = (FacetField)selectField;
if (facetField.FacetSetupDocumentId == null || facets.ContainsKey(facetField.FacetSetupDocumentId))
{
continue;
}
if (tx == null)
{
tx = context.OpenReadTransaction();
}
var documentJson = context.DocumentDatabase.DocumentsStorage.Get(context, facetField.FacetSetupDocumentId);
if (documentJson == null)
{
throw new DocumentDoesNotExistException(facetField.FacetSetupDocumentId);
}
if (facetsEtag.HasValue == false)
{
facetsEtag = documentJson.Etag;
}
else
{
facetsEtag = facetsEtag.Value ^ documentJson.Etag;
}
var document = FacetSetup.Create(facetField.FacetSetupDocumentId, documentJson.Data);
facets[facetField.FacetSetupDocumentId] = document;
}
return(new FacetQuery(query, facets, facetsEtag ?? 0)
{
Legacy = string.IsNullOrEmpty(query.ClientVersion) == false && query.ClientVersion[0] == '4'
});
}
finally
{
tx?.Dispose();
}
}
public IDisposable OpenReadTransaction()
{
var documentsTx = Documents.OpenReadTransaction();
RavenTransaction serverTx = null;
if (Server != null)
{
serverTx = Server.OpenReadTransaction();
}
return(new DisposeTransactions(documentsTx, serverTx));
}
public static FacetQuery Create(DocumentsOperationContext context, IndexQueryServerSide query)
{
long?facetsEtag = null;
DocumentsTransaction tx = null;
try
{
var facets = new Dictionary <string, FacetSetup>(StringComparer.OrdinalIgnoreCase);
foreach (var selectField in query.Metadata.SelectFields)
{
if (selectField.IsFacet == false)
{
continue;
}
var facetField = (FacetField)selectField;
if (facetField.FacetSetupDocumentId == null || facets.ContainsKey(facetField.FacetSetupDocumentId))
{
continue;
}
if (tx == null)
{
tx = context.OpenReadTransaction();
}
var documentJson = context.DocumentDatabase.DocumentsStorage.Get(context, facetField.FacetSetupDocumentId);
if (documentJson == null)
{
throw new DocumentDoesNotExistException(facetField.FacetSetupDocumentId);
}
if (facetsEtag.HasValue == false)
{
facetsEtag = documentJson.Etag;
}
else
{
facetsEtag = facetsEtag.Value ^ documentJson.Etag;
}
var document = (FacetSetup)EntityToBlittable.ConvertToEntity(typeof(FacetSetup), facetField.FacetSetupDocumentId, documentJson.Data, DocumentConventions.DefaultForServer);
facets[facetField.FacetSetupDocumentId] = document;
}
return(new FacetQuery(query, facets, facetsEtag ?? 0));
}
finally
{
tx?.Dispose();
}
}
public IDisposable Initialize(DatabaseSmugglerOptions options, SmugglerResult result, out long buildVersion)
{
_currentTypeIndex = 0;
_returnContext = _database.DocumentsStorage.ContextPool.AllocateOperationContext(out _context);
_disposeTransaction = _context.OpenReadTransaction();
buildVersion = ServerVersion.Build;
return(new DisposableAction(() =>
{
_disposeTransaction.Dispose();
_returnContext.Dispose();
}));
}
public override Task ExecuteStreamQuery(IndexQueryServerSide query, DocumentsOperationContext documentsContext, HttpResponse response, IStreamDocumentQueryResultWriter writer,
OperationCancelToken token)
{
using (var result = new StreamDocumentQueryResult(response, writer))
{
documentsContext.OpenReadTransaction();
FillCountOfResultsAndIndexEtag(result, query.Metadata, documentsContext);
ExecuteCollectionQuery(result, query, query.Metadata.CollectionName, documentsContext, token.Token);
}
return(Task.CompletedTask);
}
public async Task ExecuteStream(HttpResponse response, BlittableJsonTextWriter writer, IndexQueryServerSide query)
{
var tuple = await MatchIndex(query, false);
var index = tuple.Index;
var collection = tuple.Collection;
if (index == null)
{
using (var result = new StreamDocumentQueryResult(response, writer, _context))
{
_context.OpenReadTransaction();
FillCountOfResultsAndIndexEtag(result, collection);
ExecuteCollectionQuery(result, query, collection);
}
return;
}
await index.StreamQuery(response, writer, query, _context, _token);
}
public async Task <FacetedQueryResult> ExecuteFacetedQuery(FacetQueryServerSide query, long?facetsEtag, long?existingResultEtag, DocumentsOperationContext documentsContext, OperationCancelToken token)
{
if (query.Metadata.IsDynamic)
{
throw new InvalidQueryException("Facet query must be executed against static index.", query.Metadata.QueryText, query.QueryParameters);
}
if (query.FacetSetupDoc != null)
{
FacetSetup facetSetup;
using (documentsContext.OpenReadTransaction())
{
var facetSetupAsJson = Database.DocumentsStorage.Get(documentsContext, query.FacetSetupDoc);
if (facetSetupAsJson == null)
{
throw new DocumentDoesNotExistException(query.FacetSetupDoc);
}
try
{
facetSetup = JsonDeserializationServer.FacetSetup(facetSetupAsJson.Data);
}
catch (Exception e)
{
throw new DocumentParseException(query.FacetSetupDoc, typeof(FacetSetup), e);
}
facetsEtag = facetSetupAsJson.Etag;
}
query.Facets = facetSetup.Facets;
}
var index = GetIndex(query.Metadata.IndexName);
if (existingResultEtag.HasValue)
{
var etag = index.GetIndexEtag() ^ facetsEtag.Value;
if (etag == existingResultEtag)
{
return(FacetedQueryResult.NotModifiedResult);
}
}
return(await index.FacetedQuery(query, facetsEtag.Value, documentsContext, token));
}
private LegacySourceReplicationInformation GetSourceReplicationInformation(DocumentsOperationContext context, Guid remoteServerInstanceId, out string documentId)
{
documentId = $"Raven/Replication/Sources/{remoteServerInstanceId}";
using (context.OpenReadTransaction())
{
var document = Database.DocumentsStorage.Get(context, documentId);
if (document == null)
{
return(new LegacySourceReplicationInformation
{
ServerInstanceId = Database.DbId
});
}
return(JsonDeserializationServer.LegacySourceReplicationInformation(document.Data));
}
}
public Task <SmugglerInitializeResult> InitializeAsync(DatabaseSmugglerOptionsServerSide options, SmugglerResult result)
{
_currentTypeIndex = 0;
if (options.OperateOnTypes.HasFlag(DatabaseItemType.Documents) ||
options.OperateOnTypes.HasFlag(DatabaseItemType.RevisionDocuments) ||
options.OperateOnTypes.HasFlag(DatabaseItemType.Tombstones) ||
options.OperateOnTypes.HasFlag(DatabaseItemType.Conflicts) ||
options.OperateOnTypes.HasFlag(DatabaseItemType.CounterGroups) ||
options.OperateOnTypes.HasFlag(DatabaseItemType.TimeSeries))
{
_returnContext = _database.DocumentsStorage.ContextPool.AllocateOperationContext(out _context);
_disposeTransaction = _context.OpenReadTransaction();
LastEtag = DocumentsStorage.ReadLastEtag(_disposeTransaction.InnerTransaction);
LastDatabaseChangeVector = DocumentsStorage.GetDatabaseChangeVector(_disposeTransaction.InnerTransaction);
}
if (options.OperateOnTypes.HasFlag(DatabaseItemType.CompareExchange) ||
options.OperateOnTypes.HasFlag(DatabaseItemType.Identities) ||
options.OperateOnTypes.HasFlag(DatabaseItemType.CompareExchangeTombstones) ||
options.OperateOnTypes.HasFlag(DatabaseItemType.Subscriptions) ||
options.OperateOnTypes.HasFlag(DatabaseItemType.ReplicationHubCertificates))
{
_returnServerContext = _database.ServerStore.ContextPool.AllocateOperationContext(out _serverContext);
_disposeServerTransaction = _serverContext.OpenReadTransaction();
using (var rawRecord = _database.ServerStore.Cluster.ReadRawDatabaseRecord(_serverContext, _database.Name))
{
LastRaftIndex = rawRecord.EtagForBackup;
}
}
var disposable = new DisposableAction(() =>
{
_disposeServerTransaction?.Dispose();
_returnServerContext?.Dispose();
_disposeTransaction?.Dispose();
_returnContext?.Dispose();
});
return(Task.FromResult(new SmugglerInitializeResult(disposable, ServerVersion.Build)));
}
public override Task <DocumentQueryResult> ExecuteQuery(IndexQueryServerSide query, DocumentsOperationContext documentsContext, long?existingResultEtag, OperationCancelToken token)
{
var result = new DocumentQueryResult();
documentsContext.OpenReadTransaction();
FillCountOfResultsAndIndexEtag(result, query.Metadata, documentsContext);
if (query.Metadata.HasOrderByRandom == false && existingResultEtag.HasValue)
{
if (result.ResultEtag == existingResultEtag)
{
return(Task.FromResult(DocumentQueryResult.NotModifiedResult));
}
}
ExecuteCollectionQuery(result, query, query.Metadata.CollectionName, documentsContext, token.Token);
return(Task.FromResult(result));
}
public override Task ExecuteStreamQuery(IndexQueryServerSide query, DocumentsOperationContext documentsContext, HttpResponse response, IStreamQueryResultWriter<Document> writer,
OperationCancelToken token)
{
var result = new StreamDocumentQueryResult(response, writer, token);
documentsContext.OpenReadTransaction();
FillCountOfResultsAndIndexEtag(result, query.Metadata, documentsContext);
var collection = GetCollectionName(query.Metadata.CollectionName, out var indexName);
using (QueryRunner.MarkQueryAsRunning(indexName, query, token))
{
result.IndexName = indexName;
ExecuteCollectionQuery(result, query, collection, documentsContext, token.Token);
result.Flush();
return Task.CompletedTask;
}
}
public IDisposable Initialize(DatabaseSmugglerOptions options, SmugglerResult result, out long buildVersion)
{
_currentTypeIndex = 0;
if (options.OperateOnTypes.HasFlag(DatabaseItemType.Documents) ||
options.OperateOnTypes.HasFlag(DatabaseItemType.RevisionDocuments) ||
options.OperateOnTypes.HasFlag(DatabaseItemType.Tombstones) ||
options.OperateOnTypes.HasFlag(DatabaseItemType.Conflicts) ||
options.OperateOnTypes.HasFlag(DatabaseItemType.CounterGroups))
{
_returnContext = _database.DocumentsStorage.ContextPool.AllocateOperationContext(out _context);
_disposeTransaction = _context.OpenReadTransaction();
LastEtag = DocumentsStorage.ReadLastEtag(_disposeTransaction.InnerTransaction);
LastDatabaseChangeVector = DocumentsStorage.GetDatabaseChangeVector(_disposeTransaction.InnerTransaction);
}
if (options.OperateOnTypes.HasFlag(DatabaseItemType.CompareExchange) ||
options.OperateOnTypes.HasFlag(DatabaseItemType.Identities) ||
options.OperateOnTypes.HasFlag(DatabaseItemType.CompareExchangeTombstones))
{
_returnServerContext = _database.ServerStore.ContextPool.AllocateOperationContext(out _serverContext);
_disposeServerTransaction = _serverContext.OpenReadTransaction();
using (var rawRecord = _database.ServerStore.Cluster.ReadRawDatabaseRecord(_serverContext, _database.Name))
{
LastRaftIndex = rawRecord.GetEtagForBackup();
}
}
buildVersion = ServerVersion.Build;
return(new DisposableAction(() =>
{
_disposeServerTransaction?.Dispose();
_returnServerContext?.Dispose();
_disposeTransaction?.Dispose();
_returnContext?.Dispose();
}));
}
private long GetStartEtagForSubscription(DocumentsOperationContext docsContext, SubscriptionState subscription)
{
using (docsContext.OpenReadTransaction())
{
long startEtag = 0;
if (string.IsNullOrEmpty(subscription.ChangeVectorForNextBatchStartingPoint))
{
return(startEtag);
}
var changeVector = subscription.ChangeVectorForNextBatchStartingPoint.ToChangeVector();
var cv = changeVector.FirstOrDefault(x => x.DbId == TcpConnection.DocumentDatabase.DbBase64Id);
if (cv.DbId == "" && cv.Etag == 0 && cv.NodeTag == 0)
{
return(startEtag);
}
return(cv.Etag);
}
}
private static void FillDocumentsInfo(DatabaseStatusReport prevDatabaseReport, DocumentDatabase dbInstance, DatabaseStatusReport report,
DocumentsOperationContext context, DocumentsStorage documentsStorage)
{
if (prevDatabaseReport?.LastTransactionId != null && prevDatabaseReport.LastTransactionId == dbInstance.LastTransactionId)
{
report.LastEtag = prevDatabaseReport.LastEtag;
report.LastTombstoneEtag = prevDatabaseReport.LastTombstoneEtag;
report.NumberOfConflicts = prevDatabaseReport.NumberOfConflicts;
report.NumberOfDocuments = prevDatabaseReport.NumberOfDocuments;
report.DatabaseChangeVector = prevDatabaseReport.DatabaseChangeVector;
}
else
{
using (var tx = context.OpenReadTransaction())
{
report.LastEtag = DocumentsStorage.ReadLastEtag(tx.InnerTransaction);
report.LastTombstoneEtag = DocumentsStorage.ReadLastTombstoneEtag(tx.InnerTransaction);
report.NumberOfConflicts = documentsStorage.ConflictsStorage.ConflictsCount;
report.NumberOfDocuments = documentsStorage.GetNumberOfDocuments(context);
report.DatabaseChangeVector = DocumentsStorage.GetDatabaseChangeVector(context);
}
}
}
public virtual bool Execute(DocumentsOperationContext databaseContext, TransactionOperationContext indexContext,
Lazy <IndexWriteOperation> writeOperation, IndexingStatsScope stats, CancellationToken token)
{
const int pageSize = int.MaxValue;
var maxTimeForDocumentTransactionToRemainOpen = Debugger.IsAttached == false
? _configuration.MaxTimeForDocumentTransactionToRemainOpen.AsTimeSpan
: TimeSpan.FromMinutes(15);
var moreWorkFound = false;
foreach (var collection in _index.Collections)
{
using (var collectionStats = stats.For("Collection_" + collection))
{
var lastMappedEtag = _indexStorage.ReadLastIndexedEtag(indexContext.Transaction, collection);
var lastTombstoneEtag = _indexStorage.ReadLastProcessedTombstoneEtag(indexContext.Transaction, collection);
if (_logger.IsInfoEnabled)
{
_logger.Info($"Executing cleanup for '{_index} ({_index.Name})'. Collection: {collection}. LastMappedEtag: {lastMappedEtag:#,#;;0}. LastTombstoneEtag: {lastTombstoneEtag:#,#;;0}.");
}
var inMemoryStats = _index.GetStats(collection);
var lastEtag = lastTombstoneEtag;
var count = 0;
var sw = new Stopwatch();
IndexWriteOperation indexWriter = null;
var keepRunning = true;
var lastCollectionEtag = -1L;
while (keepRunning)
{
var batchCount = 0;
using (databaseContext.OpenReadTransaction())
{
sw.Restart();
if (lastCollectionEtag == -1)
{
lastCollectionEtag = _index.GetLastTombstoneEtagInCollection(databaseContext, collection);
}
var tombstones = collection == Constants.Documents.Collections.AllDocumentsCollection
? _documentsStorage.GetTombstonesFrom(databaseContext, lastEtag + 1, 0, pageSize)
: _documentsStorage.GetTombstonesFrom(databaseContext, collection, lastEtag + 1, 0, pageSize);
foreach (var tombstone in tombstones)
{
token.ThrowIfCancellationRequested();
if (indexWriter == null)
{
indexWriter = writeOperation.Value;
}
count++;
batchCount++;
lastEtag = tombstone.Etag;
inMemoryStats.UpdateLastEtag(lastEtag, isTombstone: true);
if (_logger.IsInfoEnabled && count % 2048 == 0)
{
_logger.Info($"Executing cleanup for '{_index.Name}'. Processed count: {count:#,#;;0} etag: {lastEtag}.");
}
if (tombstone.Type != Tombstone.TombstoneType.Document)
{
continue; // this can happen when we have '@all_docs'
}
_index.HandleDelete(tombstone, collection, indexWriter, indexContext, collectionStats);
if (CanContinueBatch(databaseContext, indexContext, collectionStats, indexWriter, lastEtag, lastCollectionEtag, batchCount) == false)
{
keepRunning = false;
break;
}
if (MapDocuments.MaybeRenewTransaction(databaseContext, sw, _configuration, ref maxTimeForDocumentTransactionToRemainOpen))
{
break;
}
}
if (batchCount == 0 || batchCount >= pageSize)
{
break;
}
}
}
if (count == 0)
{
continue;
}
if (_logger.IsInfoEnabled)
{
_logger.Info($"Executing cleanup for '{_index} ({_index.Name})'. Processed {count} tombstones in '{collection}' collection in {collectionStats.Duration.TotalMilliseconds:#,#;;0} ms.");
}
if (_index.Type.IsMap())
{
_indexStorage.WriteLastTombstoneEtag(indexContext.Transaction, collection, lastEtag);
}
else
{
_mapReduceContext.ProcessedTombstoneEtags[collection] = lastEtag;
}
moreWorkFound = true;
}
}
return(moreWorkFound);
}
public bool Execute(DocumentsOperationContext databaseContext, TransactionOperationContext indexContext,
Lazy <IndexWriteOperation> writeOperation, IndexingStatsScope stats, CancellationToken token)
{
var maxTimeForDocumentTransactionToRemainOpen = Debugger.IsAttached == false
? _configuration.MaxTimeForDocumentTransactionToRemainOpen.AsTimeSpan
: TimeSpan.FromMinutes(15);
var moreWorkFound = false;
foreach (var collection in _index.Collections)
{
using (var collectionStats = stats.For("Collection_" + collection))
{
var lastMappedEtag = _indexStorage.ReadLastIndexedEtag(indexContext.Transaction, collection);
if (_logger.IsInfoEnabled)
{
_logger.Info($"Executing map for '{_index.Name}'. Collection: {collection} LastMappedEtag: {lastMappedEtag:#,#;;0}.");
}
var inMemoryStats = _index.GetStats(collection);
var lastEtag = lastMappedEtag;
var count = 0;
var resultsCount = 0;
var pageSize = int.MaxValue;
var sw = new Stopwatch();
IndexWriteOperation indexWriter = null;
var keepRunning = true;
var lastCollectionEtag = -1L;
while (keepRunning)
{
using (databaseContext.OpenReadTransaction())
{
sw.Restart();
if (lastCollectionEtag == -1)
{
lastCollectionEtag = _index.GetLastDocumentEtagInCollection(databaseContext, collection);
}
var documents = GetDocumentsEnumerator(databaseContext, collection, lastEtag, pageSize);
using (var docsEnumerator = _index.GetMapEnumerator(documents, collection, indexContext, collectionStats, _index.Type))
{
while (true)
{
if (docsEnumerator.MoveNext(out IEnumerable mapResults) == false)
{
collectionStats.RecordMapCompletedReason("No more documents to index");
keepRunning = false;
break;
}
token.ThrowIfCancellationRequested();
if (indexWriter == null)
{
indexWriter = writeOperation.Value;
}
var current = docsEnumerator.Current;
count++;
collectionStats.RecordMapAttempt();
stats.RecordDocumentSize(current.Data.Size);
if (_logger.IsInfoEnabled && count % 8192 == 0)
{
_logger.Info($"Executing map for '{_index.Name}'. Processed count: {count:#,#;;0} etag: {lastEtag:#,#;;0}.");
}
lastEtag = current.Etag;
inMemoryStats.UpdateLastEtag(lastEtag, isTombstone: false);
try
{
var numberOfResults = _index.HandleMap(current.LowerId, current.Id, mapResults,
indexWriter, indexContext, collectionStats);
_index.MapsPerSec.MarkSingleThreaded(numberOfResults);
resultsCount += numberOfResults;
collectionStats.RecordMapSuccess();
}
catch (Exception e) when(e.IsIndexError())
{
docsEnumerator.OnError();
_index.ErrorIndexIfCriticalException(e);
collectionStats.RecordMapError();
if (_logger.IsInfoEnabled)
{
_logger.Info($"Failed to execute mapping function on '{current.Id}' for '{_index.Name}'.", e);
}
collectionStats.AddMapError(current.Id, $"Failed to execute mapping function on {current.Id}. " +
$"Exception: {e}");
}
if (CanContinueBatch(databaseContext, indexContext, collectionStats, indexWriter, lastEtag, lastCollectionEtag, count) == false)
{
keepRunning = false;
break;
}
if (count >= pageSize)
{
keepRunning = false;
break;
}
if (MaybeRenewTransaction(databaseContext, sw, _configuration, ref maxTimeForDocumentTransactionToRemainOpen))
{
break;
}
}
if (_logger.IsInfoEnabled)
{
_logger.Info($"Done executing map for '{_index.Name}'. Processed count: {count:#,#;;0} LastEtag {lastEtag}.");
}
}
}
}
if (count > 0)
{
if (_logger.IsInfoEnabled)
{
_logger.Info($"Executing map for '{_index.Name}'. Processed {count:#,#;;0} documents and {resultsCount:#,#;;0} map results in '{collection}' collection in {collectionStats.Duration.TotalMilliseconds:#,#;;0} ms.");
}
moreWorkFound = true;
}
if (_index.Type.IsMap())
{
_index.SaveLastState();
_indexStorage.WriteLastIndexedEtag(indexContext.Transaction, collection, lastEtag);
}
else
{
_mapReduceContext.ProcessedDocEtags[collection] = lastEtag;
}
}
}
return(moreWorkFound);
}
private void FillDatabaseStatistics(DatabaseStatistics stats, DocumentsOperationContext context)
{
using (context.OpenReadTransaction())
{
var indexes = Database.IndexStore.GetIndexes().ToList();
var size = Database.GetSizeOnDisk();
stats.LastDocEtag = DocumentsStorage.ReadLastDocumentEtag(context.Transaction.InnerTransaction);
stats.CountOfDocuments = Database.DocumentsStorage.GetNumberOfDocuments(context);
stats.CountOfRevisionDocuments = Database.DocumentsStorage.RevisionsStorage.GetNumberOfRevisionDocuments(context);
stats.CountOfDocumentsConflicts = Database.DocumentsStorage.ConflictsStorage.GetNumberOfDocumentsConflicts(context);
stats.CountOfTombstones = Database.DocumentsStorage.GetNumberOfTombstones(context);
stats.CountOfConflicts = Database.DocumentsStorage.ConflictsStorage.ConflictsCount;
stats.SizeOnDisk = size.Data;
stats.NumberOfTransactionMergerQueueOperations = Database.TxMerger.NumberOfQueuedOperations;
stats.TempBuffersSizeOnDisk = size.TempBuffers;
stats.CountOfCounterEntries = Database.DocumentsStorage.CountersStorage.GetNumberOfCounterEntries(context);
var attachments = Database.DocumentsStorage.AttachmentsStorage.GetNumberOfAttachments(context);
stats.CountOfAttachments = attachments.AttachmentCount;
stats.CountOfUniqueAttachments = attachments.StreamsCount;
stats.CountOfIndexes = indexes.Count;
var statsDatabaseChangeVector = DocumentsStorage.GetDatabaseChangeVector(context);
stats.DatabaseChangeVector = statsDatabaseChangeVector;
stats.DatabaseId = Database.DocumentsStorage.Environment.Base64Id;
stats.Is64Bit = !Database.DocumentsStorage.Environment.Options.ForceUsing32BitsPager && IntPtr.Size == sizeof(long);
stats.Pager = Database.DocumentsStorage.Environment.Options.DataPager.GetType().ToString();
stats.Indexes = new IndexInformation[indexes.Count];
for (var i = 0; i < indexes.Count; i++)
{
var index = indexes[i];
bool isStale;
try
{
isStale = index.IsStale(context);
}
catch (OperationCanceledException)
{
// if the index has just been removed, let us consider it stale
// until it can be safely removed from the list of indexes in the
// database
isStale = true;
}
stats.Indexes[i] = new IndexInformation
{
State = index.State,
IsStale = isStale,
Name = index.Name,
LockMode = index.Definition.LockMode,
Priority = index.Definition.Priority,
Type = index.Type,
LastIndexingTime = index.LastIndexingTime
};
if (stats.LastIndexingTime.HasValue)
{
stats.LastIndexingTime = stats.LastIndexingTime >= index.LastIndexingTime ? stats.LastIndexingTime : index.LastIndexingTime;
}
else
{
stats.LastIndexingTime = index.LastIndexingTime;
}
}
}
}
private unsafe bool HandleDocuments(ActionType actionType, DocumentsOperationContext databaseContext, TransactionOperationContext indexContext, Lazy <IndexWriteOperation> writeOperation, IndexingStatsScope stats, int pageSize, TimeSpan maxTimeForDocumentTransactionToRemainOpen, CancellationToken token)
{
var moreWorkFound = false;
Dictionary <string, long> lastIndexedEtagsByCollection = null;
foreach (var collection in _index.Collections)
{
if (_referencedCollections.TryGetValue(collection, out HashSet <CollectionName> referencedCollections) == false)
{
continue;
}
if (lastIndexedEtagsByCollection == null)
{
lastIndexedEtagsByCollection = new Dictionary <string, long>(StringComparer.OrdinalIgnoreCase);
}
if (lastIndexedEtagsByCollection.TryGetValue(collection, out long lastIndexedEtag) == false)
{
lastIndexedEtagsByCollection[collection] = lastIndexedEtag = _indexStorage.ReadLastIndexedEtag(indexContext.Transaction, collection);
}
if (lastIndexedEtag == 0) // we haven't indexed yet, so we are skipping references for now
{
continue;
}
foreach (var referencedCollection in referencedCollections)
{
using (var collectionStats = stats.For("Collection_" + referencedCollection.Name))
{
if (_logger.IsInfoEnabled)
{
_logger.Info($"Executing handle references for '{_index.Name}'. Collection: {referencedCollection.Name}. Type: {actionType}.");
}
long lastReferenceEtag;
switch (actionType)
{
case ActionType.Document:
lastReferenceEtag = _indexStorage.ReadLastProcessedReferenceEtag(indexContext.Transaction, collection, referencedCollection);
break;
case ActionType.Tombstone:
lastReferenceEtag = _indexStorage.ReadLastProcessedReferenceTombstoneEtag(indexContext.Transaction, collection, referencedCollection);
break;
default:
throw new NotSupportedException();
}
if (_logger.IsInfoEnabled)
{
_logger.Info($"Executing handle references for '{_index.Name}'. LastReferenceEtag: {lastReferenceEtag}.");
}
var lastEtag = lastReferenceEtag;
var count = 0;
var sw = new Stopwatch();
IndexWriteOperation indexWriter = null;
var keepRunning = true;
var lastCollectionEtag = -1L;
while (keepRunning)
{
var batchCount = 0;
using (databaseContext.OpenReadTransaction())
{
sw.Restart();
IEnumerable <Reference> references;
switch (actionType)
{
case ActionType.Document:
if (lastCollectionEtag == -1)
{
lastCollectionEtag = _index.GetLastDocumentEtagInCollection(databaseContext, collection);
}
references = _documentsStorage
.GetDocumentsFrom(databaseContext, referencedCollection.Name, lastEtag + 1, 0, pageSize)
.Select(document =>
{
_reference.Key = document.Id;
_reference.Etag = document.Etag;
return(_reference);
});
break;
case ActionType.Tombstone:
if (lastCollectionEtag == -1)
{
lastCollectionEtag = _index.GetLastTombstoneEtagInCollection(databaseContext, collection);
}
references = _documentsStorage
.GetTombstonesFrom(databaseContext, referencedCollection.Name, lastEtag + 1, 0, pageSize)
.Select(tombstone =>
{
_reference.Key = tombstone.LowerId;
_reference.Etag = tombstone.Etag;
return(_reference);
});
break;
default:
throw new NotSupportedException();
}
foreach (var referencedDocument in references)
{
if (_logger.IsInfoEnabled)
{
_logger.Info($"Executing handle references for '{_index.Name}'. Processing reference: {referencedDocument.Key}.");
}
lastEtag = referencedDocument.Etag;
count++;
batchCount++;
var documents = new List <Document>();
foreach (var key in _indexStorage
.GetDocumentKeysFromCollectionThatReference(collection, referencedDocument.Key, indexContext.Transaction))
{
using (DocumentIdWorker.GetLower(databaseContext.Allocator, key.Content.Ptr, key.Size, out var loweredKey))
{
// when there is conflict, we need to apply same behavior as if the document would not exist
var doc = _documentsStorage.Get(databaseContext, loweredKey, throwOnConflict: false);
if (doc != null && doc.Etag <= lastIndexedEtag)
{
documents.Add(doc);
}
}
}
using (var docsEnumerator = _index.GetMapEnumerator(documents, collection, indexContext, collectionStats))
{
while (docsEnumerator.MoveNext(out IEnumerable mapResults))
{
token.ThrowIfCancellationRequested();
var current = docsEnumerator.Current;
if (indexWriter == null)
{
indexWriter = writeOperation.Value;
}
if (_logger.IsInfoEnabled)
{
_logger.Info($"Executing handle references for '{_index.Name}'. Processing document: {current.Id}.");
}
try
{
_index.HandleMap(current.LowerId, mapResults, indexWriter, indexContext, collectionStats);
}
catch (Exception e)
{
if (_logger.IsInfoEnabled)
{
_logger.Info($"Failed to execute mapping function on '{current.Id}' for '{_index.Name}'.", e);
}
}
}
}
if (CanContinueBatch(databaseContext, indexContext, collectionStats, indexWriter, lastEtag, lastCollectionEtag, batchCount) == false)
{
keepRunning = false;
break;
}
if (MapDocuments.MaybeRenewTransaction(databaseContext, sw, _configuration, ref maxTimeForDocumentTransactionToRemainOpen))
{
break;
}
}
if (batchCount == 0 || batchCount >= pageSize)
{
break;
}
}
}
if (count == 0)
{
continue;
}
if (_logger.IsInfoEnabled)
{
_logger.Info($"Executing handle references for '{_index} ({_index.Name})'. Processed {count} references in '{referencedCollection.Name}' collection in {collectionStats.Duration.TotalMilliseconds:#,#;;0} ms.");
}
switch (actionType)
{
case ActionType.Document:
_indexStorage.WriteLastReferenceEtag(indexContext.Transaction, collection, referencedCollection, lastEtag);
break;
case ActionType.Tombstone:
_indexStorage.WriteLastReferenceTombstoneEtag(indexContext.Transaction, collection, referencedCollection, lastEtag);
break;
default:
throw new NotSupportedException();
}
moreWorkFound = true;
}
}
}
return(moreWorkFound);
}
private bool HandleDocuments(ActionType actionType, DocumentsOperationContext databaseContext, TransactionOperationContext indexContext, Lazy <IndexWriteOperation> writeOperation, IndexingStatsScope stats, int pageSize, TimeSpan maxTimeForDocumentTransactionToRemainOpen, CancellationToken token)
{
var moreWorkFound = false;
Dictionary <string, long> lastIndexedEtagsByCollection = null;
foreach (var collection in _index.Collections)
{
if (_referencedCollections.TryGetValue(collection, out HashSet <CollectionName> referencedCollections) == false)
{
continue;
}
if (lastIndexedEtagsByCollection == null)
{
lastIndexedEtagsByCollection = new Dictionary <string, long>(StringComparer.OrdinalIgnoreCase);
}
if (lastIndexedEtagsByCollection.TryGetValue(collection, out long lastIndexedEtag) == false)
{
lastIndexedEtagsByCollection[collection] = lastIndexedEtag = _indexStorage.ReadLastIndexedEtag(indexContext.Transaction, collection);
}
if (lastIndexedEtag == 0) // we haven't indexed yet, so we are skipping references for now
{
continue;
}
var totalProcessedCount = 0;
foreach (var referencedCollection in referencedCollections)
{
var inMemoryStats = _index.GetReferencesStats(referencedCollection.Name);
using (var collectionStats = stats.For("Collection_" + referencedCollection.Name))
{
long lastReferenceEtag;
switch (actionType)
{
case ActionType.Document:
lastReferenceEtag = IndexStorage.ReadLastProcessedReferenceEtag(indexContext.Transaction.InnerTransaction, collection, referencedCollection);
break;
case ActionType.Tombstone:
lastReferenceEtag = IndexStorage.ReadLastProcessedReferenceTombstoneEtag(indexContext.Transaction.InnerTransaction, collection, referencedCollection);
break;
default:
throw new NotSupportedException();
}
var lastEtag = lastReferenceEtag;
var resultsCount = 0;
var sw = new Stopwatch();
var keepRunning = true;
var lastCollectionEtag = -1L;
while (keepRunning)
{
var hasChanges = false;
using (databaseContext.OpenReadTransaction())
{
sw.Restart();
IEnumerable <Reference> references;
switch (actionType)
{
case ActionType.Document:
if (lastCollectionEtag == -1)
{
lastCollectionEtag = _index.GetLastDocumentEtagInCollection(databaseContext, collection);
}
references = _documentsStorage
.GetDocumentsFrom(databaseContext, referencedCollection.Name, lastEtag + 1, 0, pageSize,
DocumentFields.Id | DocumentFields.Etag)
.Select(document =>
{
_reference.Key = document.Id;
_reference.Etag = document.Etag;
return(_reference);
});
break;
case ActionType.Tombstone:
if (lastCollectionEtag == -1)
{
lastCollectionEtag = _index.GetLastTombstoneEtagInCollection(databaseContext, collection);
}
references = _documentsStorage
.GetTombstonesFrom(databaseContext, referencedCollection.Name, lastEtag + 1, 0, pageSize)
.Select(tombstone =>
{
_reference.Key = tombstone.LowerId;
_reference.Etag = tombstone.Etag;
return(_reference);
});
break;
default:
throw new NotSupportedException();
}
var isTombstone = actionType == ActionType.Tombstone;
foreach (var referencedDocument in references)
{
lastEtag = referencedDocument.Etag;
hasChanges = true;
inMemoryStats.UpdateLastEtag(lastEtag, isTombstone);
var documents = GetDocumentFromCollectionThatReference(databaseContext, indexContext, collection, referencedDocument, lastIndexedEtag);
using (var docsEnumerator = _index.GetMapEnumerator(documents, collection, indexContext, collectionStats, _index.Type))
{
while (docsEnumerator.MoveNext(out IEnumerable mapResults))
{
token.ThrowIfCancellationRequested();
totalProcessedCount++;
collectionStats.RecordMapReferenceAttempt();
var current = docsEnumerator.Current;
stats.RecordDocumentSize(current.Data.Size);
try
{
var numberOfResults = _index.HandleMap(current.LowerId, current.Id, mapResults, writeOperation, indexContext, collectionStats);
resultsCount += numberOfResults;
collectionStats.RecordMapReferenceSuccess();
_index.MapsPerSec.MarkSingleThreaded(numberOfResults);
}
catch (Exception e) when(e.IsIndexError())
{
docsEnumerator.OnError();
_index.ErrorIndexIfCriticalException(e);
collectionStats.RecordMapReferenceError();
if (_logger.IsInfoEnabled)
{
_logger.Info($"Failed to execute mapping function on '{current.Id}' for '{_index.Name}'.", e);
}
collectionStats.AddMapReferenceError(current.Id,
$"Failed to execute mapping function on {current.Id}. Exception: {e}");
}
_index.UpdateThreadAllocations(indexContext, writeOperation, stats, updateReduceStats: false);
}
}
if (CanContinueBatch(databaseContext, indexContext, collectionStats, writeOperation, lastEtag, lastCollectionEtag, totalProcessedCount) == false)
{
keepRunning = false;
break;
}
if (totalProcessedCount >= pageSize)
{
keepRunning = false;
break;
}
if (MapDocuments.MaybeRenewTransaction(databaseContext, sw, _configuration, ref maxTimeForDocumentTransactionToRemainOpen))
{
break;
}
}
if (hasChanges == false)
{
break;
}
}
}
if (lastReferenceEtag == lastEtag)
{
// the last referenced etag hasn't changed
continue;
}
moreWorkFound = true;
if (_logger.IsInfoEnabled)
{
_logger.Info($"Executed handle references for '{_index.Name}' index and '{referencedCollection.Name}' collection. " +
$"Got {resultsCount:#,#;;0} map results in {collectionStats.Duration.TotalMilliseconds:#,#;;0} ms.");
}
switch (actionType)
{
case ActionType.Document:
_indexStorage.WriteLastReferenceEtag(indexContext.Transaction, collection, referencedCollection, lastEtag);
break;
case ActionType.Tombstone:
_indexStorage.WriteLastReferenceTombstoneEtag(indexContext.Transaction, collection, referencedCollection, lastEtag);
break;
default:
throw new NotSupportedException();
}
}
}
}
return(moreWorkFound);
}
protected async Task <IOperationResult> ExecuteOperation(string collectionName, long start, long take, CollectionOperationOptions options, DocumentsOperationContext context,
Action <DeterminateProgress> onProgress, Func <string, TransactionOperationsMerger.MergedTransactionCommand> action, OperationCancelToken token)
{
var progress = new DeterminateProgress();
var cancellationToken = token.Token;
var isAllDocs = collectionName == Constants.Documents.Collections.AllDocumentsCollection;
long lastEtag;
long totalCount;
using (context.OpenReadTransaction())
{
lastEtag = GetLastEtagForCollection(context, collectionName, isAllDocs);
totalCount = GetTotalCountForCollection(context, collectionName, isAllDocs);
}
progress.Total = totalCount;
// send initial progress with total count set, and 0 as processed count
onProgress(progress);
long startEtag = 0;
var alreadySeenIdsCount = new Reference <long>();
string startAfterId = null;
using (var rateGate = options.MaxOpsPerSecond.HasValue
? new RateGate(options.MaxOpsPerSecond.Value, TimeSpan.FromSeconds(1))
: null)
{
var end = false;
var ids = new Queue <string>(OperationBatchSize);
while (startEtag <= lastEtag)
{
cancellationToken.ThrowIfCancellationRequested();
ids.Clear();
Database.ForTestingPurposes?.CollectionRunnerBeforeOpenReadTransaction?.Invoke();
using (context.OpenReadTransaction())
{
foreach (var document in GetDocuments(context, collectionName, startEtag, startAfterId, alreadySeenIdsCount, OperationBatchSize, isAllDocs, DocumentFields.Id, out bool isStartsWithOrIdQuery))
{
using (document)
{
cancellationToken.ThrowIfCancellationRequested();
token.Delay();
if (isAllDocs && document.Id.StartsWith(HiLoHandler.RavenHiloIdPrefix, StringComparison.OrdinalIgnoreCase))
{
continue;
}
// start with and id queries aren't ordered by the etag
if (isStartsWithOrIdQuery == false && document.Etag > lastEtag)
{
// we don't want to go over the documents that we have patched
end = true;
break;
}
startEtag = document.Etag + 1;
if (start > 0)
{
start--;
continue;
}
if (take-- <= 0)
{
end = true;
break;
}
startAfterId = document.Id;
ids.Enqueue(document.Id);
context.Transaction.ForgetAbout(document);
}
}
}
if (ids.Count == 0)
{
break;
}
do
{
var command = new ExecuteRateLimitedOperations <string>(ids, action, rateGate, token,
maxTransactionSize: 16 * Voron.Global.Constants.Size.Megabyte,
batchSize: OperationBatchSize);
await Database.TxMerger.Enqueue(command);
progress.Processed += command.Processed;
onProgress(progress);
if (command.NeedWait)
{
rateGate?.WaitToProceed();
}
token.Delay();
} while (ids.Count > 0);
if (end)
{
break;
}
}
}
return(new BulkOperationResult
{
Total = progress.Processed
});
}
private async Task <(List <Match> Matches, GraphQueryPlan QueryPlan, bool NotModified)> GetQueryResults(IndexQueryServerSide query, DocumentsOperationContext documentsContext, long?existingResultEtag, OperationCancelToken token, bool collectIntermediateResults = false)
{
var q = query.Metadata.Query;
var qp = new GraphQueryPlan(query, documentsContext, existingResultEtag, token, Database)
{
CollectIntermediateResults = collectIntermediateResults
};
qp.BuildQueryPlan();
qp.OptimizeQueryPlan(); //TODO: audit optimization
if (query.WaitForNonStaleResults)
{
qp.IsStale = await qp.WaitForNonStaleResults();
}
else
{
await qp.CreateAutoIndexesAndWaitIfNecessary();
}
//for the case where we don't wait for non stale results we will override IsStale in the QueryQueryStep steps
if (documentsContext.Transaction == null || documentsContext.Transaction.Disposed)
{
documentsContext.OpenReadTransaction();
}
qp.ResultEtag = DocumentsStorage.ReadLastEtag(documentsContext.Transaction.InnerTransaction);
if (existingResultEtag.HasValue)
{
if (qp.ResultEtag == existingResultEtag)
{
return(null, null, true);
}
}
await qp.Initialize();
var matchResults = qp.Execute();
if (query.Metadata.OrderBy != null)
{
Sort(matchResults, query.Metadata.OrderBy, Database.Name, query.Query);
}
var filter = q.GraphQuery.Where;
if (filter != null)
{
for (int i = 0; i < matchResults.Count; i++)
{
var resultAsJson = new DynamicJsonValue();
matchResults[i].PopulateVertices(resultAsJson);
using (var result = documentsContext.ReadObject(resultAsJson, "graph/result"))
{
if (filter.IsMatchedBy(result, query.QueryParameters) == false)
{
matchResults[i] = default;
}
}
}
}
if (query.Start > 0)
{
matchResults.RemoveRange(0, Math.Min(query.Start, matchResults.Count));
}
if (query.PageSize < matchResults.Count)
{
matchResults.RemoveRange(query.PageSize, matchResults.Count - query.PageSize);
}
return(matchResults, qp, false);
}
public SuggestionQueryResultServerSide ExecuteSuggestionQuery(SuggestionQueryServerSide query, DocumentsOperationContext context, long?existingResultEtag, OperationCancelToken token)
{
if (query == null)
{
throw new ArgumentNullException(nameof(query));
}
// Check pre-requisites for the query to happen.
if (string.IsNullOrWhiteSpace(query.Term))
{
throw new InvalidOperationException("Suggestions queries require a term.");
}
if (string.IsNullOrWhiteSpace(query.Field))
{
throw new InvalidOperationException("Suggestions queries require a field.");
}
var sw = Stopwatch.StartNew();
// Check definition for the index.
var index = GetIndex(query.IndexName);
var indexDefinition = index.GetIndexDefinition();
if (indexDefinition == null)
{
throw new InvalidOperationException($"Could not find specified index '{this}'.");
}
if (indexDefinition.Fields.TryGetValue(query.Field, out IndexFieldOptions field) == false)
{
throw new InvalidOperationException($"Index '{this}' does not have a field '{query.Field}'.");
}
if (field.Suggestions == null)
{
throw new InvalidOperationException($"Index '{this}' does not have suggestions configured for field '{query.Field}'.");
}
if (field.Suggestions.Value == false)
{
throw new InvalidOperationException($"Index '{this}' have suggestions explicitly disabled for field '{query.Field}'.");
}
if (existingResultEtag.HasValue)
{
var etag = index.GetIndexEtag();
if (etag == existingResultEtag.Value)
{
return(SuggestionQueryResultServerSide.NotModifiedResult);
}
}
context.OpenReadTransaction();
var result = index.SuggestionsQuery(query, context, token);
result.DurationInMs = (int)sw.Elapsed.TotalMilliseconds;
return(result);
}
protected async Task <IOperationResult> ExecuteOperation(string collectionName, CollectionOperationOptions options, DocumentsOperationContext context,
Action <DeterminateProgress> onProgress, Func <LazyStringValue, TransactionOperationsMerger.MergedTransactionCommand> action, OperationCancelToken token)
{
const int batchSize = 1024;
var progress = new DeterminateProgress();
var cancellationToken = token.Token;
var isAllDocs = collectionName == Constants.Documents.Collections.AllDocumentsCollection;
long lastEtag;
long totalCount;
using (context.OpenReadTransaction())
{
lastEtag = GetLastEtagForCollection(context, collectionName, isAllDocs);
totalCount = GetTotalCountForCollection(context, collectionName, isAllDocs);
}
progress.Total = totalCount;
// send initial progress with total count set, and 0 as processed count
onProgress(progress);
long startEtag = 0;
using (var rateGate = options.MaxOpsPerSecond.HasValue
? new RateGate(options.MaxOpsPerSecond.Value, TimeSpan.FromSeconds(1))
: null)
{
var end = false;
while (startEtag <= lastEtag)
{
cancellationToken.ThrowIfCancellationRequested();
using (context.OpenReadTransaction())
{
var ids = new Queue <LazyStringValue>(batchSize);
foreach (var document in GetDocuments(context, collectionName, startEtag, batchSize, isAllDocs))
{
cancellationToken.ThrowIfCancellationRequested();
token.Delay();
if (isAllDocs && IsHiLoDocument(document))
{
continue;
}
if (document.Etag > lastEtag) // we don't want to go over the documents that we have patched
{
end = true;
break;
}
startEtag = document.Etag + 1;
ids.Enqueue(document.Id);
}
if (ids.Count == 0)
{
break;
}
do
{
var command = new ExecuteRateLimitedOperations <LazyStringValue>(ids, action, rateGate, token);
await Database.TxMerger.Enqueue(command);
progress.Processed += command.Processed;
onProgress(progress);
if (command.NeedWait)
{
rateGate?.WaitToProceed();
}
} while (ids.Count > 0);
if (end)
{
break;
}
}
}
}
return(new BulkOperationResult
{
Total = progress.Processed
});
}