private unsafe void FillCountOfResultsAndIndexEtag(QueryResultServerSide <Document> resultToFill, QueryMetadata query, DocumentsOperationContext context) { var collection = query.CollectionName; var buffer = stackalloc long[3]; // If the query has include or load, it's too difficult to check the etags for just the included collections, // it's easier to just show etag for all docs instead. if (collection == Constants.Documents.Collections.AllDocumentsCollection || query.HasIncludeOrLoad) { var numberOfDocuments = Database.DocumentsStorage.GetNumberOfDocuments(context); buffer[0] = DocumentsStorage.ReadLastDocumentEtag(context.Transaction.InnerTransaction); buffer[1] = DocumentsStorage.ReadLastTombstoneEtag(context.Transaction.InnerTransaction); buffer[2] = numberOfDocuments; resultToFill.TotalResults = (int)numberOfDocuments; } else { var collectionStats = Database.DocumentsStorage.GetCollection(collection, context); buffer[0] = Database.DocumentsStorage.GetLastDocumentEtag(context, collection); buffer[1] = Database.DocumentsStorage.GetLastTombstoneEtag(context, collection); buffer[2] = collectionStats.Count; resultToFill.TotalResults = (int)collectionStats.Count; } resultToFill.ResultEtag = (long)Hashing.XXHash64.Calculate((byte *)buffer, sizeof(long) * 3); resultToFill.NodeTag = Database.ServerStore.NodeTag; }
private void ExecuteCollectionQuery(QueryResultServerSide resultToFill, IndexQueryServerSide query, string collection) { var isAllDocsCollection = collection == Constants.Documents.Collections.AllDocumentsCollection; // we optimize for empty queries without sorting options, appending CollectionIndexPrefix to be able to distinguish index for collection vs. physical index resultToFill.IndexName = isAllDocsCollection ? "AllDocs" : CollectionIndexPrefix + collection; resultToFill.IsStale = false; resultToFill.LastQueryTime = DateTime.MinValue; resultToFill.IndexTimestamp = DateTime.MinValue; resultToFill.IncludedPaths = query.Metadata.Includes; var includeDocumentsCommand = new IncludeDocumentsCommand(_documents, _context, query.Metadata.Includes); var fieldsToFetch = new FieldsToFetch(query, null); var totalResults = new Reference <int>(); var documents = new CollectionQueryEnumerable(_database, _documents, fieldsToFetch, collection, query, _context, includeDocumentsCommand, totalResults); var cancellationToken = _token.Token; try { foreach (var document in documents) { cancellationToken.ThrowIfCancellationRequested(); resultToFill.AddResult(document); includeDocumentsCommand.Gather(document); } } catch (Exception e) { if (resultToFill.SupportsExceptionHandling == false) { throw; } resultToFill.HandleException(e); } includeDocumentsCommand.Fill(resultToFill.Includes); resultToFill.TotalResults = totalResults.Value; }
private unsafe void FillCountOfResultsAndIndexEtag(QueryResultServerSide resultToFill, string collection, DocumentsOperationContext context) { var buffer = stackalloc long[3]; if (collection == Constants.Documents.Collections.AllDocumentsCollection) { var numberOfDocuments = Database.DocumentsStorage.GetNumberOfDocuments(context); buffer[0] = DocumentsStorage.ReadLastDocumentEtag(context.Transaction.InnerTransaction); buffer[1] = DocumentsStorage.ReadLastTombstoneEtag(context.Transaction.InnerTransaction); buffer[2] = numberOfDocuments; resultToFill.TotalResults = (int)numberOfDocuments; } else { var collectionStats = Database.DocumentsStorage.GetCollection(collection, context); buffer[0] = Database.DocumentsStorage.GetLastDocumentEtag(context, collection); buffer[1] = Database.DocumentsStorage.GetLastTombstoneEtag(context, collection); buffer[2] = collectionStats.Count; } resultToFill.ResultEtag = (long)Hashing.XXHash64.Calculate((byte *)buffer, sizeof(long) * 3); }
private void ExecuteCollectionQuery(QueryResultServerSide <Document> resultToFill, IndexQueryServerSide query, string collection, DocumentsOperationContext context, CancellationToken cancellationToken) { using (var queryScope = query.Timings?.For(nameof(QueryTimingsScope.Names.Query))) { QueryTimingsScope gatherScope = null; QueryTimingsScope fillScope = null; if (queryScope != null && query.Metadata.Includes?.Length > 0) { var includesScope = queryScope.For(nameof(QueryTimingsScope.Names.Includes), start: false); gatherScope = includesScope.For(nameof(QueryTimingsScope.Names.Gather), start: false); fillScope = includesScope.For(nameof(QueryTimingsScope.Names.Fill), start: false); } var isAllDocsCollection = collection == Constants.Documents.Collections.AllDocumentsCollection; // we optimize for empty queries without sorting options, appending CollectionIndexPrefix to be able to distinguish index for collection vs. physical index resultToFill.IndexName = isAllDocsCollection ? "AllDocs" : CollectionIndexPrefix + collection; resultToFill.IsStale = false; resultToFill.LastQueryTime = DateTime.MinValue; resultToFill.IndexTimestamp = DateTime.MinValue; resultToFill.IncludedPaths = query.Metadata.Includes; var fieldsToFetch = new FieldsToFetch(query, null); var includeDocumentsCommand = new IncludeDocumentsCommand(Database.DocumentsStorage, context, query.Metadata.Includes, fieldsToFetch.IsProjection); var totalResults = new Reference <int>(); var documents = new CollectionQueryEnumerable(Database, Database.DocumentsStorage, fieldsToFetch, collection, query, queryScope, context, includeDocumentsCommand, totalResults); IncludeCountersCommand includeCountersCommand = null; if (query.Metadata.HasCounters) { includeCountersCommand = new IncludeCountersCommand( Database, context, query.Metadata.CounterIncludes.Counters); } try { foreach (var document in documents) { cancellationToken.ThrowIfCancellationRequested(); resultToFill.AddResult(document); using (gatherScope?.Start()) includeDocumentsCommand.Gather(document); includeCountersCommand?.Fill(document); } } catch (Exception e) { if (resultToFill.SupportsExceptionHandling == false) { throw; } resultToFill.HandleException(e); } using (fillScope?.Start()) includeDocumentsCommand.Fill(resultToFill.Includes); if (includeCountersCommand != null) { resultToFill.AddCounterIncludes(includeCountersCommand); } resultToFill.TotalResults = (totalResults.Value == 0 && resultToFill.Results.Count != 0) ? -1 : totalResults.Value; } }
private unsafe void FillCountOfResultsAndIndexEtag(QueryResultServerSide <Document> resultToFill, QueryMetadata query, DocumentsOperationContext context) { var bufferSize = 3; var hasCounters = query.HasCounterSelect || query.CounterIncludes != null; if (hasCounters) { bufferSize++; } if (query.HasCmpXchgSelect) { bufferSize++; } var collection = query.CollectionName; var buffer = stackalloc long[bufferSize]; // If the query has include or load, it's too difficult to check the etags for just the included collections, // it's easier to just show etag for all docs instead. if (collection == Constants.Documents.Collections.AllDocumentsCollection || query.HasIncludeOrLoad) { var numberOfDocuments = Database.DocumentsStorage.GetNumberOfDocuments(context); buffer[0] = DocumentsStorage.ReadLastDocumentEtag(context.Transaction.InnerTransaction); buffer[1] = DocumentsStorage.ReadLastTombstoneEtag(context.Transaction.InnerTransaction); buffer[2] = numberOfDocuments; if (hasCounters) { buffer[3] = DocumentsStorage.ReadLastCountersEtag(context.Transaction.InnerTransaction); } resultToFill.TotalResults = (int)numberOfDocuments; } else { var collectionStats = Database.DocumentsStorage.GetCollection(collection, context); buffer[0] = Database.DocumentsStorage.GetLastDocumentEtag(context, collection); buffer[1] = Database.DocumentsStorage.GetLastTombstoneEtag(context, collection); buffer[2] = collectionStats.Count; if (hasCounters) { buffer[3] = Database.DocumentsStorage.CountersStorage.GetLastCounterEtag(context, collection); } resultToFill.TotalResults = (int)collectionStats.Count; } if (query.HasCmpXchgSelect) { using (context.DocumentDatabase.ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext transactionContext)) using (transactionContext.OpenReadTransaction()) { buffer[bufferSize - 1] = Database.ServerStore.Cluster.GetLastCompareExchangeIndexForDatabase(transactionContext, Database.Name); } } resultToFill.ResultEtag = (long)Hashing.XXHash64.Calculate((byte *)buffer, sizeof(long) * (uint)bufferSize); resultToFill.NodeTag = Database.ServerStore.NodeTag; }
private void ExecuteCollectionQuery(QueryResultServerSide <Document> resultToFill, IndexQueryServerSide query, string collection, DocumentsOperationContext context, bool pulseReadingTransaction, CancellationToken cancellationToken) { using (var queryScope = query.Timings?.For(nameof(QueryTimingsScope.Names.Query))) { QueryTimingsScope gatherScope = null; QueryTimingsScope fillScope = null; if (queryScope != null && query.Metadata.Includes?.Length > 0) { var includesScope = queryScope.For(nameof(QueryTimingsScope.Names.Includes), start: false); gatherScope = includesScope.For(nameof(QueryTimingsScope.Names.Gather), start: false); fillScope = includesScope.For(nameof(QueryTimingsScope.Names.Fill), start: false); } // we optimize for empty queries without sorting options, appending CollectionIndexPrefix to be able to distinguish index for collection vs. physical index resultToFill.IsStale = false; resultToFill.LastQueryTime = DateTime.MinValue; resultToFill.IndexTimestamp = DateTime.MinValue; resultToFill.IncludedPaths = query.Metadata.Includes; var fieldsToFetch = new FieldsToFetch(query, null); var includeDocumentsCommand = new IncludeDocumentsCommand(Database.DocumentsStorage, context, query.Metadata.Includes, fieldsToFetch.IsProjection); var totalResults = new Reference <int>(); IEnumerator <Document> enumerator; if (pulseReadingTransaction == false) { var documents = new CollectionQueryEnumerable(Database, Database.DocumentsStorage, fieldsToFetch, collection, query, queryScope, context, includeDocumentsCommand, totalResults); enumerator = documents.GetEnumerator(); } else { enumerator = new PulsedTransactionEnumerator <Document, CollectionQueryResultsIterationState>(context, state => { query.Start = state.Start; query.PageSize = state.Take; var documents = new CollectionQueryEnumerable(Database, Database.DocumentsStorage, fieldsToFetch, collection, query, queryScope, context, includeDocumentsCommand, totalResults); return(documents); }, new CollectionQueryResultsIterationState(context, Database.Configuration.Databases.PulseReadTransactionLimit) { Start = query.Start, Take = query.PageSize }); } IncludeCountersCommand includeCountersCommand = null; if (query.Metadata.CounterIncludes != null) { includeCountersCommand = new IncludeCountersCommand( Database, context, query.Metadata.CounterIncludes.Counters); } try { using (enumerator) { while (enumerator.MoveNext()) { var document = enumerator.Current; cancellationToken.ThrowIfCancellationRequested(); resultToFill.AddResult(document); using (gatherScope?.Start()) includeDocumentsCommand.Gather(document); includeCountersCommand?.Fill(document); } } } catch (Exception e) { if (resultToFill.SupportsExceptionHandling == false) { throw; } resultToFill.HandleException(e); } using (fillScope?.Start()) includeDocumentsCommand.Fill(resultToFill.Includes); if (includeCountersCommand != null) { resultToFill.AddCounterIncludes(includeCountersCommand); } resultToFill.TotalResults = (totalResults.Value == 0 && resultToFill.Results.Count != 0) ? -1 : totalResults.Value; if (query.Offset != null || query.Limit != null) { if (resultToFill.TotalResults == -1) { resultToFill.CappedMaxResults = query.Limit ?? -1; } else { resultToFill.CappedMaxResults = Math.Min( query.Limit ?? int.MaxValue, resultToFill.TotalResults - (query.Offset ?? 0) ); } } } }
private void ExecuteCollectionQuery(QueryResultServerSide resultToFill, IndexQueryServerSide query, string collection) { var isAllDocsCollection = collection == Constants.Indexing.AllDocumentsCollection; // we optimize for empty queries without sorting options resultToFill.IndexName = isAllDocsCollection ? "AllDocs" : collection; resultToFill.IsStale = false; resultToFill.ResultEtag = Environment.TickCount; resultToFill.LastQueryTime = DateTime.MinValue; resultToFill.IndexTimestamp = DateTime.MinValue; _context.OpenReadTransaction(); if (isAllDocsCollection) { resultToFill.TotalResults = (int)_documents.GetNumberOfDocuments(_context); } else { var collectionStats = _documents.GetCollection(collection, _context); resultToFill.TotalResults = (int)collectionStats.Count; } var includeDocumentsCommand = new IncludeDocumentsCommand(_documents, _context, query.Includes); Transformer transformer = null; if (string.IsNullOrEmpty(query.Transformer) == false) { transformer = _transformerStore.GetTransformer(query.Transformer); if (transformer == null) { throw new InvalidOperationException($"The transformer '{query.Transformer}' was not found."); } } using (var scope = transformer?.OpenTransformationScope(query.TransformerParameters, includeDocumentsCommand, _documents, _transformerStore, _context)) { var fieldsToFetch = new FieldsToFetch(query, null, transformer); var documents = new CollectionQueryEnumerable(_documents, fieldsToFetch, collection, query, _context); var results = scope != null?scope.Transform(documents) : documents; try { foreach (var document in results) { _token.Token.ThrowIfCancellationRequested(); resultToFill.AddResult(document); includeDocumentsCommand.Gather(document); } } catch (Exception e) { if (resultToFill.SupportsExceptionHandling == false) { throw; } resultToFill.HandleException(e); } } includeDocumentsCommand.Fill(resultToFill.Includes); }