private QueryExecutor CreateQueryExecutor(IQueryResultRetriever retrieverMock) { return new QueryExecutor (_resolverStub, retrieverMock, ResultOperatorHandlerRegistry.CreateDefault (), CompoundMethodCallTransformerProvider.CreateDefault (), false); }
public IEnumerable <Document> MoreLikeThis( IndexQueryServerSide query, IQueryResultRetriever retriever, DocumentsOperationContext context, CancellationToken token) { var moreLikeThisQuery = QueryBuilder.BuildMoreLikeThisQuery(context, query.Metadata, query.Metadata.Query.Where, query.QueryParameters, _analyzer, _queryBuilderFactories); var options = moreLikeThisQuery.Options != null?JsonDeserializationServer.MoreLikeThisOptions(moreLikeThisQuery.Options) : MoreLikeThisOptions.Default; HashSet <string> stopWords = null; if (string.IsNullOrWhiteSpace(options.StopWordsDocumentId) == false) { var stopWordsDoc = context.DocumentDatabase.DocumentsStorage.Get(context, options.StopWordsDocumentId); if (stopWordsDoc == null) { throw new InvalidOperationException($"Stop words document {options.StopWordsDocumentId} could not be found"); } if (stopWordsDoc.Data.TryGet(nameof(MoreLikeThisStopWords.StopWords), out BlittableJsonReaderArray value) && value != null) { stopWords = new HashSet <string>(StringComparer.OrdinalIgnoreCase); for (var i = 0; i < value.Length; i++) { stopWords.Add(value.GetStringByIndex(i)); } } } var ir = _searcher.IndexReader; var mlt = new RavenMoreLikeThis(ir, options, _state); int?baseDocId = null; if (moreLikeThisQuery.BaseDocument == null) { var td = _searcher.Search(moreLikeThisQuery.BaseDocumentQuery, 1, _state); // get the current Lucene docid for the given RavenDB doc ID if (td.ScoreDocs.Length == 0) { throw new InvalidOperationException("Given filtering expression did not yield any documents that could be used as a base of comparison"); } baseDocId = td.ScoreDocs[0].Doc; } if (stopWords != null) { mlt.SetStopWords(stopWords); } string[] fieldNames; if (options.Fields != null && options.Fields.Length > 0) { fieldNames = options.Fields; } else { fieldNames = ir.GetFieldNames(IndexReader.FieldOption.INDEXED) .Where(x => x != Constants.Documents.Indexing.Fields.DocumentIdFieldName && x != Constants.Documents.Indexing.Fields.ReduceKeyHashFieldName) .ToArray(); } mlt.SetFieldNames(fieldNames); mlt.Analyzer = _analyzer; var pageSize = GetPageSize(_searcher, query.PageSize); Query mltQuery; if (baseDocId.HasValue) { mltQuery = mlt.Like(baseDocId.Value); } else { using (var blittableJson = ParseJsonStringIntoBlittable(moreLikeThisQuery.BaseDocument, context)) mltQuery = mlt.Like(blittableJson); } var tsdc = TopScoreDocCollector.Create(pageSize, true); if (moreLikeThisQuery.FilterQuery != null && moreLikeThisQuery.FilterQuery is MatchAllDocsQuery == false) { mltQuery = new BooleanQuery { { mltQuery, Occur.MUST }, { moreLikeThisQuery.FilterQuery, Occur.MUST } }; } _searcher.Search(mltQuery, tsdc, _state); var hits = tsdc.TopDocs().ScoreDocs; var ids = new HashSet <string>(StringComparer.OrdinalIgnoreCase); foreach (var hit in hits) { if (hit.Doc == baseDocId) { continue; } var doc = _searcher.Doc(hit.Doc, _state); var id = doc.Get(Constants.Documents.Indexing.Fields.DocumentIdFieldName, _state) ?? doc.Get(Constants.Documents.Indexing.Fields.ReduceKeyHashFieldName, _state); if (id == null) { continue; } if (ids.Add(id) == false) { continue; } yield return(retriever.Get(doc, hit.Score, _state)); } }
public IEnumerable <Document> Query(IndexQueryServerSide query, FieldsToFetch fieldsToFetch, Reference <int> totalResults, Reference <int> skippedResults, IQueryResultRetriever retriever, JsonOperationContext documentsContext, Func <string, SpatialField> getSpatialField, CancellationToken token) { var pageSize = GetPageSize(_searcher, query.PageSize); var docsToGet = pageSize; var position = query.Start; var luceneQuery = GetLuceneQuery(documentsContext, query.Metadata, query.QueryParameters, _analyzer, _queryBuilderFactories); var sort = GetSort(query, getSpatialField); var returnedResults = 0; using (var scope = new IndexQueryingScope(_indexType, query, fieldsToFetch, _searcher, retriever, _state)) { while (true) { token.ThrowIfCancellationRequested(); var search = ExecuteQuery(luceneQuery, query.Start, docsToGet, sort); totalResults.Value = search.TotalHits; scope.RecordAlreadyPagedItemsInPreviousPage(search); for (; position < search.ScoreDocs.Length && pageSize > 0; position++) { token.ThrowIfCancellationRequested(); var scoreDoc = search.ScoreDocs[position]; var document = _searcher.Doc(scoreDoc.Doc, _state); if (retriever.TryGetKey(document, _state, out string key) && scope.WillProbablyIncludeInResults(key) == false) { skippedResults.Value++; continue; } var result = retriever.Get(document, scoreDoc.Score, _state); if (scope.TryIncludeInResults(result) == false) { skippedResults.Value++; continue; } returnedResults++; yield return(result); if (returnedResults == pageSize) { yield break; } } docsToGet += GetPageSize(_searcher, (long)(pageSize - returnedResults) * _maxNumberOfOutputsPerDocument); if (search.TotalHits == search.ScoreDocs.Length) { break; } if (returnedResults >= pageSize) { break; } } } }
public IEnumerable <Document> IntersectQuery(IndexQueryServerSide query, FieldsToFetch fieldsToFetch, Reference <int> totalResults, Reference <int> skippedResults, IQueryResultRetriever retriever, JsonOperationContext documentsContext, Func <string, SpatialField> getSpatialField, CancellationToken token) { var method = query.Metadata.Query.Where as MethodExpression; if (method == null) { throw new InvalidQueryException($"Invalid intersect query. WHERE clause must contains just an intersect() method call while it got {query.Metadata.Query.Where.Type} expression", query.Metadata.QueryText, query.QueryParameters); } var methodName = method.Name; if (string.Equals("intersect", methodName) == false) { throw new InvalidQueryException($"Invalid intersect query. WHERE clause must contains just a single intersect() method call while it got '{methodName}' method", query.Metadata.QueryText, query.QueryParameters); } if (method.Arguments.Count <= 1) { throw new InvalidQueryException("The valid intersect query must have multiple intersect clauses.", query.Metadata.QueryText, query.QueryParameters); } var subQueries = new Query[method.Arguments.Count]; for (var i = 0; i < subQueries.Length; i++) { var whereExpression = method.Arguments[i] as QueryExpression; if (whereExpression == null) { throw new InvalidQueryException($"Invalid intersect query. The intersect clause at position {i} isn't a valid expression", query.Metadata.QueryText, query.QueryParameters); } subQueries[i] = GetLuceneQuery(documentsContext, query.Metadata, whereExpression, query.QueryParameters, _analyzer, _queryBuilderFactories); } //Not sure how to select the page size here??? The problem is that only docs in this search can be part //of the final result because we're doing an intersection query (but we might exclude some of them) var pageSize = GetPageSize(_searcher, query.PageSize); int pageSizeBestGuess = GetPageSize(_searcher, ((long)query.Start + query.PageSize) * 2); int skippedResultsInCurrentLoop = 0; int previousBaseQueryMatches = 0; var firstSubDocumentQuery = subQueries[0]; var sort = GetSort(query, getSpatialField); using (var scope = new IndexQueryingScope(_indexType, query, fieldsToFetch, _searcher, retriever, _state)) { //Do the first sub-query in the normal way, so that sorting, filtering etc is accounted for var search = ExecuteQuery(firstSubDocumentQuery, 0, pageSizeBestGuess, sort); var currentBaseQueryMatches = search.ScoreDocs.Length; var intersectionCollector = new IntersectionCollector(_searcher, search.ScoreDocs, _state); int intersectMatches; do { token.ThrowIfCancellationRequested(); if (skippedResultsInCurrentLoop > 0) { // We get here because out first attempt didn't get enough docs (after INTERSECTION was calculated) pageSizeBestGuess = pageSizeBestGuess * 2; search = ExecuteQuery(firstSubDocumentQuery, 0, pageSizeBestGuess, sort); previousBaseQueryMatches = currentBaseQueryMatches; currentBaseQueryMatches = search.ScoreDocs.Length; intersectionCollector = new IntersectionCollector(_searcher, search.ScoreDocs, _state); } for (var i = 1; i < subQueries.Length; i++) { _searcher.Search(subQueries[i], null, intersectionCollector, _state); } var currentIntersectResults = intersectionCollector.DocumentsIdsForCount(subQueries.Length).ToList(); intersectMatches = currentIntersectResults.Count; skippedResultsInCurrentLoop = pageSizeBestGuess - intersectMatches; } while (intersectMatches < pageSize && //stop if we've got enough results to satisfy the pageSize currentBaseQueryMatches < search.TotalHits && //stop if increasing the page size wouldn't make any difference previousBaseQueryMatches < currentBaseQueryMatches); //stop if increasing the page size didn't result in any more "base query" results var intersectResults = intersectionCollector.DocumentsIdsForCount(subQueries.Length).ToList(); //It's hard to know what to do here, the TotalHits from the base search isn't really the TotalSize, //because it's before the INTERSECTION has been applied, so only some of those results make it out. //Trying to give an accurate answer is going to be too costly, so we aren't going to try. totalResults.Value = search.TotalHits; skippedResults.Value = skippedResultsInCurrentLoop; //Using the final set of results in the intersectionCollector int returnedResults = 0; for (int i = query.Start; i < intersectResults.Count && (i - query.Start) < pageSizeBestGuess; i++) { var indexResult = intersectResults[i]; var document = _searcher.Doc(indexResult.LuceneId, _state); if (retriever.TryGetKey(document, _state, out string key) && scope.WillProbablyIncludeInResults(key) == false) { skippedResults.Value++; skippedResultsInCurrentLoop++; continue; } var result = retriever.Get(document, indexResult.Score, _state); if (scope.TryIncludeInResults(result) == false) { skippedResults.Value++; skippedResultsInCurrentLoop++; continue; } returnedResults++; yield return(result); if (returnedResults == pageSize) { yield break; } } } }