public override Task <DocumentQueryResult> ExecuteQuery(IndexQueryServerSide query, QueryOperationContext queryContext, long?existingResultEtag, OperationCancelToken token)
{
var result = new DocumentQueryResult();
if (queryContext.AreTransactionsOpened() == false)
{
queryContext.OpenReadTransaction();
}
FillCountOfResultsAndIndexEtag(result, query.Metadata, queryContext);
if (query.Metadata.HasOrderByRandom == false && existingResultEtag.HasValue)
{
if (result.ResultEtag == existingResultEtag)
{
return(Task.FromResult(DocumentQueryResult.NotModifiedResult));
}
}
var collection = GetCollectionName(query.Metadata.CollectionName, out var indexName);
using (QueryRunner.MarkQueryAsRunning(indexName, query, token))
{
result.IndexName = indexName;
ExecuteCollectionQuery(result, query, collection, queryContext, pulseReadingTransaction: false, token.Token);
return(Task.FromResult(result));
}
}
public async Task CreateAutoIndexesAndWaitIfNecessary()
{
var queryStepsGatherer = new QueryQueryStepGatherer();
queryStepsGatherer.Visit(RootQueryStep);
if (_context.AreTransactionsOpened() == false)
{
_context.OpenReadTransaction();
}
try
{
var etag = DocumentsStorage.ReadLastEtag(_context.Documents.Transaction.InnerTransaction);
var queryDuration = Stopwatch.StartNew();
var indexes = new List <Index>();
var indexWaiters = new Dictionary <Index, (IndexQueryServerSide, AsyncWaitForIndexing)>();
foreach (var queryStepInfo in queryStepsGatherer.QuerySteps)
{
if (string.IsNullOrWhiteSpace(queryStepInfo.QueryStep.Query.From.From.FieldValue) || queryStepInfo.IsIndexQuery)
{
continue;
}
var indexQuery = new IndexQueryServerSide(queryStepInfo.QueryStep.GetQueryString, queryStepInfo.QueryStep.QueryParameters);
//No sense creating an index for collection queries
if (indexQuery.Metadata.IsCollectionQuery)
{
continue;
}
var indexCreationInfo = await _dynamicQueryRunner.CreateAutoIndexIfNeeded(indexQuery, true, null, _database.DatabaseShutdown);
if (indexCreationInfo.HasCreatedAutoIndex) //wait for non-stale only IF we just created an auto-index
{
indexes.Add(indexCreationInfo.Index);
var queryTimeout = indexQuery.WaitForNonStaleResultsTimeout ?? Index.DefaultWaitForNonStaleResultsTimeout;
indexWaiters.Add(indexCreationInfo.Index, (indexQuery, new AsyncWaitForIndexing(queryDuration, queryTimeout, indexCreationInfo.Index)));
}
}
await WaitForNonStaleResultsInternal(etag, indexes, indexWaiters);
}
finally
{
//The rest of the code assumes that a Tx is not opened
_context.CloseTransaction();
}
}
public override async Task ExecuteStreamQuery(IndexQueryServerSide query, QueryOperationContext queryContext, HttpResponse response, IStreamQueryResultWriter <Document> writer,
OperationCancelToken token)
{
var result = new StreamDocumentQueryResult(response, writer, token);
using (queryContext.OpenReadTransaction())
{
FillCountOfResultsAndIndexEtag(result, query.Metadata, queryContext);
var collection = GetCollectionName(query.Metadata.CollectionName, out var indexName);
using (QueryRunner.MarkQueryAsRunning(indexName, query, token, true))
{
result.IndexName = indexName;
await ExecuteCollectionQueryAsync(result, query, collection, queryContext, pulseReadingTransaction : true, token.Token);
result.Flush();
}
}
}
public bool Execute(QueryOperationContext queryContext, TransactionOperationContext indexContext,
Lazy <IndexWriteOperation> writeOperation, IndexingStatsScope stats, CancellationToken token)
{
var maxTimeForDocumentTransactionToRemainOpen = Debugger.IsAttached == false
? _configuration.MaxTimeForDocumentTransactionToRemainOpen.AsTimeSpan
: TimeSpan.FromMinutes(15);
var moreWorkFound = false;
var totalProcessedCount = 0;
foreach (var collection in _index.Collections)
{
using (var collectionStats = stats.For("Collection_" + collection))
{
var lastMappedEtag = _indexStorage.ReadLastIndexedEtag(indexContext.Transaction, collection);
if (_logger.IsInfoEnabled)
{
_logger.Info($"Executing map for '{_index.Name}'. Collection: {collection} LastMappedEtag: {lastMappedEtag:#,#;;0}.");
}
var inMemoryStats = _index.GetStats(collection);
var lastEtag = lastMappedEtag;
var resultsCount = 0;
var pageSize = int.MaxValue;
var sw = new Stopwatch();
IndexWriteOperation indexWriter = null;
var keepRunning = true;
var lastCollectionEtag = -1L;
while (keepRunning)
{
using (queryContext.OpenReadTransaction())
{
sw.Restart();
if (lastCollectionEtag == -1)
{
lastCollectionEtag = _index.GetLastItemEtagInCollection(queryContext, collection);
}
var items = GetItemsEnumerator(queryContext, collection, lastEtag, pageSize);
using (var itemEnumerator = _index.GetMapEnumerator(items, collection, indexContext, collectionStats, _index.Type))
{
while (true)
{
if (itemEnumerator.MoveNext(queryContext.Documents, out IEnumerable mapResults, out var etag) == false)
{
if (etag > lastEtag)
{
lastEtag = etag.Value;
}
collectionStats.RecordMapCompletedReason("No more documents to index");
keepRunning = false;
break;
}
token.ThrowIfCancellationRequested();
if (indexWriter == null)
{
indexWriter = writeOperation.Value;
}
var current = itemEnumerator.Current;
totalProcessedCount++;
collectionStats.RecordMapAttempt();
stats.RecordDocumentSize(current.Size);
if (_logger.IsInfoEnabled && totalProcessedCount % 8192 == 0)
{
_logger.Info($"Executing map for '{_index.Name}'. Processed count: {totalProcessedCount:#,#;;0} etag: {lastEtag:#,#;;0}.");
}
lastEtag = current.Etag;
inMemoryStats.UpdateLastEtag(lastEtag, isTombstone: false);
try
{
var numberOfResults = _index.HandleMap(current, mapResults,
indexWriter, indexContext, collectionStats);
resultsCount += numberOfResults;
collectionStats.RecordMapSuccess();
_index.MapsPerSec?.MarkSingleThreaded(numberOfResults);
}
catch (Exception e) when(e.IsIndexError())
{
itemEnumerator.OnError();
_index.ErrorIndexIfCriticalException(e);
collectionStats.RecordMapError();
if (_logger.IsInfoEnabled)
{
_logger.Info($"Failed to execute mapping function on '{current.Id}' for '{_index.Name}'.", e);
}
collectionStats.AddMapError(current.Id, $"Failed to execute mapping function on {current.Id}. " +
$"Exception: {e}");
}
if (CanContinueBatch(queryContext, indexContext, collectionStats, indexWriter, lastEtag, lastCollectionEtag, totalProcessedCount) == false)
{
keepRunning = false;
break;
}
if (totalProcessedCount >= pageSize)
{
keepRunning = false;
break;
}
if (MaybeRenewTransaction(queryContext, sw, _configuration, ref maxTimeForDocumentTransactionToRemainOpen))
{
break;
}
}
}
}
}
if (lastMappedEtag == lastEtag)
{
// the last mapped etag hasn't changed
continue;
}
moreWorkFound = true;
if (_logger.IsInfoEnabled)
{
_logger.Info($"Executed map for '{_index.Name}' index and '{collection}' collection. Got {resultsCount:#,#;;0} map results in {collectionStats.Duration.TotalMilliseconds:#,#;;0} ms.");
}
if (_index.Type.IsMap())
{
_index.SaveLastState();
_indexStorage.WriteLastIndexedEtag(indexContext.Transaction, collection, lastEtag);
}
else
{
_mapReduceContext.ProcessedDocEtags[collection] = lastEtag;
}
}
}
return(moreWorkFound);
}
public virtual bool Execute(QueryOperationContext queryContext, TransactionOperationContext indexContext,
Lazy <IndexWriteOperation> writeOperation, IndexingStatsScope stats, CancellationToken token)
{
const long pageSize = long.MaxValue;
var maxTimeForDocumentTransactionToRemainOpen = Debugger.IsAttached == false
? _configuration.MaxTimeForDocumentTransactionToRemainOpen.AsTimeSpan
: TimeSpan.FromMinutes(15);
var moreWorkFound = false;
var totalProcessedCount = 0;
foreach (var collection in _index.Collections)
{
using (var collectionStats = stats.For("Collection_" + collection))
{
var lastMappedEtag = _indexStorage.ReadLastIndexedEtag(indexContext.Transaction, collection);
var lastTombstoneEtag = _indexStorage.ReadLastProcessedTombstoneEtag(indexContext.Transaction, collection);
if (_logger.IsInfoEnabled)
{
_logger.Info($"Executing cleanup for '{_index} ({_index.Name})'. Collection: {collection}. LastMappedEtag: {lastMappedEtag:#,#;;0}. LastTombstoneEtag: {lastTombstoneEtag:#,#;;0}.");
}
var inMemoryStats = _index.GetStats(collection);
var lastEtag = lastTombstoneEtag;
var count = 0;
var sw = new Stopwatch();
var keepRunning = true;
var lastCollectionEtag = -1L;
while (keepRunning)
{
var hasChanges = false;
using (queryContext.OpenReadTransaction())
{
sw.Restart();
if (lastCollectionEtag == -1)
{
lastCollectionEtag = _index.GetLastTombstoneEtagInCollection(queryContext, collection);
}
var tombstones = collection == Constants.Documents.Collections.AllDocumentsCollection
? _documentsStorage.GetTombstonesFrom(queryContext.Documents, lastEtag + 1, 0, pageSize)
: _documentsStorage.GetTombstonesFrom(queryContext.Documents, collection, lastEtag + 1, 0, pageSize);
foreach (var tombstone in tombstones)
{
token.ThrowIfCancellationRequested();
count++;
totalProcessedCount++;
hasChanges = true;
lastEtag = tombstone.Etag;
inMemoryStats.UpdateLastEtag(lastEtag, isTombstone: true);
if (_logger.IsInfoEnabled && totalProcessedCount % 2048 == 0)
{
_logger.Info($"Executing cleanup for '{_index.Name}'. Processed count: {totalProcessedCount:#,#;;0} etag: {lastEtag}.");
}
if (tombstone.Type != Tombstone.TombstoneType.Document)
{
continue; // this can happen when we have '@all_docs'
}
_index.HandleDelete(tombstone, collection, writeOperation, indexContext, collectionStats);
var canContinueBatch = _index.CanContinueBatch(stats, queryContext, indexContext, writeOperation, lastEtag, lastCollectionEtag,
totalProcessedCount, sw, ref maxTimeForDocumentTransactionToRemainOpen);
if (canContinueBatch != Index.CanContinueBatchResult.True)
{
keepRunning = canContinueBatch == Index.CanContinueBatchResult.RenewTransaction;
break;
}
}
if (hasChanges == false)
{
break;
}
}
}
if (count == 0)
{
continue;
}
if (_logger.IsInfoEnabled)
{
_logger.Info($"Executing cleanup for '{_index} ({_index.Name})'. Processed {count} tombstones in '{collection}' collection in {collectionStats.Duration.TotalMilliseconds:#,#;;0} ms.");
}
if (_index.Type.IsMap())
{
_indexStorage.WriteLastTombstoneEtag(indexContext.Transaction, collection, lastEtag);
}
else
{
_mapReduceContext.ProcessedTombstoneEtags[collection] = lastEtag;
}
moreWorkFound = true;
}
}
return(moreWorkFound);
}
private void FillDatabaseStatistics(DatabaseStatistics stats, QueryOperationContext context)
{
using (context.OpenReadTransaction())
{
var indexes = Database.IndexStore.GetIndexes().ToList();
var size = Database.GetSizeOnDisk();
stats.LastDocEtag = DocumentsStorage.ReadLastDocumentEtag(context.Documents.Transaction.InnerTransaction);
stats.LastDatabaseEtag = DocumentsStorage.ReadLastEtag(context.Documents.Transaction.InnerTransaction);
stats.DatabaseChangeVector = DocumentsStorage.GetDatabaseChangeVector(context.Documents);
stats.CountOfDocuments = Database.DocumentsStorage.GetNumberOfDocuments(context.Documents);
stats.CountOfRevisionDocuments = Database.DocumentsStorage.RevisionsStorage.GetNumberOfRevisionDocuments(context.Documents);
stats.CountOfDocumentsConflicts = Database.DocumentsStorage.ConflictsStorage.GetNumberOfDocumentsConflicts(context.Documents);
stats.CountOfTombstones = Database.DocumentsStorage.GetNumberOfTombstones(context.Documents);
stats.CountOfConflicts = Database.DocumentsStorage.ConflictsStorage.ConflictsCount;
stats.SizeOnDisk = size.Data;
stats.NumberOfTransactionMergerQueueOperations = Database.TxMerger.NumberOfQueuedOperations;
stats.TempBuffersSizeOnDisk = size.TempBuffers;
stats.CountOfCounterEntries = Database.DocumentsStorage.CountersStorage.GetNumberOfCounterEntries(context.Documents);
stats.CountOfTimeSeriesSegments = Database.DocumentsStorage.TimeSeriesStorage.GetNumberOfTimeSeriesSegments(context.Documents);
var attachments = Database.DocumentsStorage.AttachmentsStorage.GetNumberOfAttachments(context.Documents);
stats.CountOfAttachments = attachments.AttachmentCount;
stats.CountOfUniqueAttachments = attachments.StreamsCount;
stats.CountOfIndexes = indexes.Count;
stats.DatabaseId = Database.DocumentsStorage.Environment.Base64Id;
stats.Is64Bit = !Database.DocumentsStorage.Environment.Options.ForceUsing32BitsPager && IntPtr.Size == sizeof(long);
stats.Pager = Database.DocumentsStorage.Environment.Options.DataPager.GetType().ToString();
stats.Indexes = new IndexInformation[indexes.Count];
for (var i = 0; i < indexes.Count; i++)
{
var index = indexes[i];
bool isStale;
try
{
isStale = index.IsStale(context);
}
catch (OperationCanceledException)
{
// if the index has just been removed, let us consider it stale
// until it can be safely removed from the list of indexes in the
// database
isStale = true;
}
stats.Indexes[i] = new IndexInformation
{
State = index.State,
IsStale = isStale,
Name = index.Name,
LockMode = index.Definition.LockMode,
Priority = index.Definition.Priority,
Type = index.Type,
LastIndexingTime = index.LastIndexingTime,
SourceType = index.SourceType
};
if (stats.LastIndexingTime.HasValue)
{
stats.LastIndexingTime = stats.LastIndexingTime >= index.LastIndexingTime ? stats.LastIndexingTime : index.LastIndexingTime;
}
else
{
stats.LastIndexingTime = index.LastIndexingTime;
}
}
}
}
private async Task <(List <Match> Matches, GraphQueryPlan QueryPlan, bool NotModified)> GetQueryResults(IndexQueryServerSide query, QueryOperationContext queryContext, long?existingResultEtag, OperationCancelToken token, bool collectIntermediateResults = false)
{
var q = query.Metadata.Query;
var qp = new GraphQueryPlan(query, queryContext, existingResultEtag, token, Database)
{
CollectIntermediateResults = collectIntermediateResults
};
qp.BuildQueryPlan();
qp.OptimizeQueryPlan(); //TODO: audit optimization
if (query.WaitForNonStaleResults)
{
qp.IsStale = await qp.WaitForNonStaleResults();
}
else
{
await qp.CreateAutoIndexesAndWaitIfNecessary();
}
//for the case where we don't wait for non stale results we will override IsStale in the QueryQueryStep steps
if (queryContext.AreTransactionsOpened() == false)
{
queryContext.OpenReadTransaction();
}
qp.ResultEtag = DocumentsStorage.ReadLastEtag(queryContext.Documents.Transaction.InnerTransaction);
if (existingResultEtag.HasValue)
{
if (qp.ResultEtag == existingResultEtag)
{
return(null, null, true);
}
}
await qp.Initialize();
var matchResults = qp.Execute();
if (query.Metadata.OrderBy != null)
{
Sort(matchResults, query.Metadata.OrderBy, Database.Name, query.Query);
}
var filter = q.GraphQuery.Where;
if (filter != null)
{
for (int i = 0; i < matchResults.Count; i++)
{
var resultAsJson = new DynamicJsonValue();
matchResults[i].PopulateVertices(resultAsJson);
using (var result = queryContext.Documents.ReadObject(resultAsJson, "graph/result"))
{
if (filter.IsMatchedBy(result, query.QueryParameters) == false)
{
matchResults[i] = default;
}
}
}
}
if (query.Start > 0)
{
matchResults.RemoveRange(0, Math.Min(query.Start, matchResults.Count));
}
if (query.PageSize < matchResults.Count)
{
matchResults.RemoveRange(query.PageSize, matchResults.Count - query.PageSize);
}
return(matchResults, qp, false);
}