public virtual void TestComplexNested() { PayloadNearQuery query; TopDocs hits; // combine ordered and unordered spans with some nesting to make sure all payloads are counted SpanQuery q1 = NewPhraseQuery("field", "nine hundred", true); SpanQuery q2 = NewPhraseQuery("field", "ninety nine", true); SpanQuery q3 = NewPhraseQuery("field", "nine ninety", false); SpanQuery q4 = NewPhraseQuery("field", "hundred nine", false); SpanQuery[] clauses = new SpanQuery[] { new PayloadNearQuery(new SpanQuery[] { q1, q2 }, 0, true), new PayloadNearQuery(new SpanQuery[] { q3, q4 }, 0, false) }; query = new PayloadNearQuery(clauses, 0, false); hits = searcher.Search(query, null, 100); Assert.IsTrue(hits != null, "hits is null and it shouldn't be"); // should be only 1 hit - doc 999 Assert.IsTrue(hits.ScoreDocs.Length == 1, "should only be one hit"); // the score should be 3 - the average of all the underlying payloads ScoreDoc doc = hits.ScoreDocs[0]; // System.out.println("Doc: " + doc.toString()); // System.out.println("Explain: " + searcher.explain(query, doc.doc)); Assert.IsTrue(doc.Score == 3, doc.Score + " does not equal: " + 3); }
public virtual void Test() { BoostingTermQuery query = new BoostingTermQuery(new Term("field", "seventy")); TopDocs hits = searcher.Search(query, null, 100); Assert.IsTrue(hits != null, "hits is null and it shouldn't be"); Assert.IsTrue(hits.TotalHits == 100, "hits Size: " + hits.TotalHits + " is not: " + 100); //they should all have the exact same score, because they all contain seventy once, and we set //all the other similarity factors to be 1 Assert.IsTrue(hits.GetMaxScore() == 1, hits.GetMaxScore() + " does not equal: " + 1); for (int i = 0; i < hits.ScoreDocs.Length; i++) { ScoreDoc doc = hits.ScoreDocs[i]; Assert.IsTrue(doc.score == 1, doc.score + " does not equal: " + 1); } CheckHits.CheckExplanations(query, PayloadHelper.FIELD, searcher, true); Lucene.Net.Search.Spans.Spans spans = query.GetSpans(searcher.GetIndexReader()); Assert.IsTrue(spans != null, "spans is null and it shouldn't be"); Assert.IsTrue(spans is TermSpans, "spans is not an instanceof " + typeof(TermSpans)); /*float score = hits.score(0); * for (int i =1; i < hits.length(); i++) * { * Assert.IsTrue(score == hits.score(i), "scores are not equal and they should be"); * }*/ }
protected Document ReturnProjection(DynamicJsonValue result, Document doc, Lucene.Net.Search.ScoreDoc scoreDoc, JsonOperationContext context) { result[Constants.Documents.Metadata.Key] = new DynamicJsonValue { [Constants.Documents.Metadata.Projection] = true }; var newData = context.ReadObject(result, "projection result"); try { if (ReferenceEquals(newData, doc.Data) == false) { doc.Data?.Dispose(); } } catch (Exception) { newData.Dispose(); throw; } doc.Data = newData; FinishDocumentSetup(doc, scoreDoc); return(doc); }
public virtual void Test() { PayloadNearQuery query; TopDocs hits; query = NewPhraseQuery("field", "twenty two", true); QueryUtils.Check(query); // all 10 hits should have score = 3 because adjacent terms have payloads of 2,4 // and all the similarity factors are set to 1 hits = searcher.Search(query, null, 100); Assert.IsTrue(hits != null, "hits is null and it shouldn't be"); Assert.IsTrue(hits.TotalHits == 10, "should be 10 hits"); for (int j = 0; j < hits.ScoreDocs.Length; j++) { ScoreDoc doc = hits.ScoreDocs[j]; Assert.IsTrue(doc.Score == 3, doc.Score + " does not equal: " + 3); } for (int i = 1; i < 10; i++) { query = NewPhraseQuery("field", English.IntToEnglish(i) + " hundred", true); // all should have score = 3 because adjacent terms have payloads of 2,4 // and all the similarity factors are set to 1 hits = searcher.Search(query, null, 100); Assert.IsTrue(hits != null, "hits is null and it shouldn't be"); Assert.IsTrue(hits.TotalHits == 100, "should be 100 hits"); for (int j = 0; j < hits.ScoreDocs.Length; j++) { ScoreDoc doc = hits.ScoreDocs[j]; // System.out.println("Doc: " + doc.toString()); // System.out.println("Explain: " + searcher.explain(query, doc.doc)); Assert.IsTrue(doc.Score == 3, doc.Score + " does not equal: " + 3); } } }
protected void FinishDocumentSetup(Document doc, Lucene.Net.Search.ScoreDoc scoreDoc) { if (doc == null || scoreDoc == null) { return; } doc.IndexScore = scoreDoc.Score; if (_query?.Distances != null) { doc.Distance = _query.Distances.Get(scoreDoc.Doc); } }
public virtual void TestIgnoreSpanScorer() { PayloadTermQuery query = new PayloadTermQuery(new Term(PayloadHelper.MULTI_FIELD, "seventy"), new MaxPayloadFunction(), false); IndexSearcher theSearcher = new IndexSearcher(directory, true, null); theSearcher.Similarity = new FullSimilarity(); TopDocs hits = searcher.Search(query, null, 100, null); Assert.IsTrue(hits != null, "hits is null and it shouldn't be"); Assert.IsTrue(hits.TotalHits == 100, "hits Size: " + hits.TotalHits + " is not: " + 100); //they should all have the exact same score, because they all contain seventy once, and we set //all the other similarity factors to be 1 //System.out.println("Hash: " + seventyHash + " Twice Hash: " + 2*seventyHash); Assert.IsTrue(hits.MaxScore == 4.0, hits.MaxScore + " does not equal: " + 4.0); //there should be exactly 10 items that score a 4, all the rest should score a 2 //The 10 items are: 70 + i*100 where i in [0-9] int numTens = 0; for (int i = 0; i < hits.ScoreDocs.Length; i++) { ScoreDoc doc = hits.ScoreDocs[i]; if (doc.Doc % 10 == 0) { numTens++; Assert.IsTrue(doc.Score == 4.0, doc.Score + " does not equal: " + 4.0); } else { Assert.IsTrue(doc.Score == 2, doc.Score + " does not equal: " + 2); } } Assert.IsTrue(numTens == 10, numTens + " does not equal: " + 10); CheckHits.CheckExplanations(query, "field", searcher, true); Lucene.Net.Search.Spans.Spans spans = query.GetSpans(searcher.IndexReader, null); Assert.IsTrue(spans != null, "spans is null and it shouldn't be"); Assert.IsTrue(spans is TermSpans, "spans is not an instanceof " + typeof(TermSpans)); //should be two matches per document int count = 0; //100 hits times 2 matches per hit, we should have 200 in count while (spans.Next(null)) { count++; } }
public virtual void TestLongerSpan() { PayloadNearQuery query; TopDocs hits; query = NewPhraseQuery("field", "nine hundred ninety nine", true); hits = searcher.Search(query, null, 100); ScoreDoc doc = hits.ScoreDocs[0]; // System.out.println("Doc: " + doc.toString()); // System.out.println("Explain: " + searcher.explain(query, doc.doc)); Assert.IsTrue(hits != null, "hits is null and it shouldn't be"); Assert.IsTrue(hits.TotalHits == 1, "there should only be one hit"); // should have score = 3 because adjacent terms have payloads of 2,4 Assert.IsTrue(doc.Score == 3, doc.Score + " does not equal: " + 3); }
public void Query(string searchText) { //state the file location of the index DirectoryInfo directoryInfo = new DirectoryInfo(_Location); Lucene.Net.Store.Directory dir = Lucene.Net.Store.FSDirectory.Open(directoryInfo); //create an index searcher that will perform the search Lucene.Net.Search.IndexSearcher searcher = new Lucene.Net.Search.IndexSearcher(dir, true); //build a query object Lucene.Net.Index.Term searchTerm = new Lucene.Net.Index.Term("content", searchText); Lucene.Net.Search.Query query = new Lucene.Net.Search.TermQuery(searchTerm); //execute the query Lucene.Net.Search.TopDocs hits = searcher.Search(query, null, 100); //iterate over the results. for (int i = 0; i < hits.TotalHits; i++) { Lucene.Net.Search.ScoreDoc doc = hits.ScoreDocs[i]; } }
public override Document Get(Lucene.Net.Documents.Document input, Lucene.Net.Search.ScoreDoc scoreDoc, IState state) { using (RetrieverScope?.Start()) { if (TryGetKey(input, state, out string id) == false) { throw new InvalidOperationException($"Could not extract '{Constants.Documents.Indexing.Fields.DocumentIdFieldName}' from index."); } if (FieldsToFetch.IsProjection) { return(GetProjection(input, scoreDoc, id, state)); } using (_storageScope = _storageScope?.Start() ?? RetrieverScope?.For(nameof(QueryTimingsScope.Names.Storage))) { var doc = DirectGet(null, id, DocumentFields, state); FinishDocumentSetup(doc, scoreDoc); return(doc); } } }
protected Document AddProjectionToResult(Document doc, Lucene.Net.Search.ScoreDoc scoreDoc, FieldsToFetch fieldsToFetch, DynamicJsonValue result, string key, object fieldVal) { if (fieldsToFetch.SingleBodyOrMethodWithNoAlias) { Document newDoc = null; if (fieldVal is BlittableJsonReaderObject nested) { newDoc = new Document { Id = doc.Id, ChangeVector = doc.ChangeVector, Data = nested, Etag = doc.Etag, Flags = doc.Flags, LastModified = doc.LastModified, LowerId = doc.LowerId, NonPersistentFlags = doc.NonPersistentFlags, StorageId = doc.StorageId, TransactionMarker = doc.TransactionMarker }; } else if (fieldVal is Document d) { newDoc = d; } else { ThrowInvalidQueryBodyResponse(fieldVal); } FinishDocumentSetup(newDoc, scoreDoc); return(newDoc); } AddProjectionToResult(result, key, fieldVal); return(null); }
public override (Document Document, List <Document> List) Get(Lucene.Net.Documents.Document input, Lucene.Net.Search.ScoreDoc scoreDoc, IState state, CancellationToken token) { throw new NotSupportedException("Graph Queries do not deal with Lucene indexes."); }
private Document GetProjectionFromDocumentInternal(Document doc, Lucene.Net.Documents.Document luceneDoc, Lucene.Net.Search.ScoreDoc scoreDoc, FieldsToFetch fieldsToFetch, JsonOperationContext context, IState state) { var result = new DynamicJsonValue(); foreach (var fieldToFetch in fieldsToFetch.Fields.Values) { if (TryGetValue(fieldToFetch, doc, luceneDoc, state, fieldsToFetch.IndexFields, fieldsToFetch.AnyDynamicIndexFields, out var key, out var fieldVal) == false && fieldToFetch.QueryField != null && fieldToFetch.QueryField.HasSourceAlias) { continue; } var immediateResult = AddProjectionToResult(doc, scoreDoc, fieldsToFetch, result, key, fieldVal); if (immediateResult != null) { return(immediateResult); } } return(ReturnProjection(result, doc, scoreDoc, context)); }
public Document GetProjectionFromDocument(Document doc, Lucene.Net.Documents.Document luceneDoc, Lucene.Net.Search.ScoreDoc scoreDoc, FieldsToFetch fieldsToFetch, JsonOperationContext context, IState state) { using (RetrieverScope?.Start()) using (_projectionScope = _projectionScope?.Start() ?? RetrieverScope?.For(nameof(QueryTimingsScope.Names.Projection))) { return(GetProjectionFromDocumentInternal(doc, luceneDoc, scoreDoc, fieldsToFetch, context, state)); } }
protected (Document Document, List <Document> List) GetProjection(Lucene.Net.Documents.Document input, Lucene.Net.Search.ScoreDoc scoreDoc, string lowerId, IState state, CancellationToken token) { using (_projectionScope = _projectionScope?.Start() ?? RetrieverScope?.For(nameof(QueryTimingsScope.Names.Projection))) { Document doc = null; if (FieldsToFetch.AnyExtractableFromIndex == false) { using (_projectionStorageScope = _projectionStorageScope?.Start() ?? _projectionScope?.For(nameof(QueryTimingsScope.Names.Storage))) doc = DirectGet(input, lowerId, DocumentFields.All, state); if (doc == null) { if (FieldsToFetch.Projection.MustExtractFromDocument) { if (FieldsToFetch.Projection.MustExtractOrThrow) { FieldsToFetch.Projection.ThrowCouldNotExtractProjectionOnDocumentBecauseDocumentDoesNotExistException(lowerId); } } return(default);
public abstract Document Get(Lucene.Net.Documents.Document input, Lucene.Net.Search.ScoreDoc scoreDoc, IState state);
public override Document Get(Lucene.Net.Documents.Document input, Lucene.Net.Search.ScoreDoc scoreDoc, IState state) { throw new NotSupportedException("Graph Queries do not deal with Lucene indexes."); }
private void TestHits(ScoreDoc[] hits, int expectedCount, IndexReader reader) { int hitCount = hits.Length; Assert.AreEqual(expectedCount, hitCount, "wrong number of hits"); for (int i = 0; i < hitCount; i++) { reader.Document(hits[i].doc); reader.GetTermFreqVectors(hits[i].doc); } }
protected Document GetProjection(Lucene.Net.Documents.Document input, Lucene.Net.Search.ScoreDoc scoreDoc, string lowerId, IState state) { using (_projectionScope = _projectionScope?.Start() ?? RetrieverScope?.For(nameof(QueryTimingsScope.Names.Projection))) { Document doc = null; if (FieldsToFetch.AnyExtractableFromIndex == false) { using (_projectionStorageScope = _projectionStorageScope?.Start() ?? _projectionScope?.For(nameof(QueryTimingsScope.Names.Storage))) doc = DirectGet(input, lowerId, DocumentFields.All, state); if (doc == null) { return(null); } return(GetProjectionFromDocumentInternal(doc, input, scoreDoc, FieldsToFetch, _context, state)); } var documentLoaded = false; var result = new DynamicJsonValue(); Dictionary <string, FieldsToFetch.FieldToFetch> fields = null; if (FieldsToFetch.ExtractAllFromIndex) { fields = input.GetFields() .Where(x => x.Name != Constants.Documents.Indexing.Fields.DocumentIdFieldName && x.Name != Constants.Documents.Indexing.Fields.SourceDocumentIdFieldName && x.Name != Constants.Documents.Indexing.Fields.ReduceKeyHashFieldName && x.Name != Constants.Documents.Indexing.Fields.ReduceKeyValueFieldName && x.Name != Constants.Documents.Indexing.Fields.ValueFieldName && FieldUtil.GetRangeTypeFromFieldName(x.Name) == RangeType.None) .Distinct(UniqueFieldNames.Instance) .ToDictionary(x => x.Name, x => new FieldsToFetch.FieldToFetch(x.Name, null, null, x.IsStored, isDocumentId: false, isTimeSeries: false)); } if (fields == null) { fields = FieldsToFetch.Fields; } else if (FieldsToFetch.Fields != null && FieldsToFetch.Fields.Count > 0) { foreach (var kvp in FieldsToFetch.Fields) { if (fields.ContainsKey(kvp.Key)) { continue; } fields[kvp.Key] = kvp.Value; } } foreach (var fieldToFetch in fields.Values) { if (TryExtractValueFromIndex(fieldToFetch, input, result, state)) { continue; } if (documentLoaded == false) { using (_projectionStorageScope = _projectionStorageScope?.Start() ?? _projectionScope?.For(nameof(QueryTimingsScope.Names.Storage))) doc = DirectGet(input, lowerId, DocumentFields.All, state); documentLoaded = true; } if (doc == null) { continue; } if (TryGetValue(fieldToFetch, doc, input, state, FieldsToFetch.IndexFields, FieldsToFetch.AnyDynamicIndexFields, out var key, out var fieldVal)) { if (FieldsToFetch.SingleBodyOrMethodWithNoAlias) { if (fieldVal is BlittableJsonReaderObject nested) { doc.Data = nested; } else if (fieldVal is Document d) { doc = d; } else { ThrowInvalidQueryBodyResponse(fieldVal); } FinishDocumentSetup(doc, scoreDoc); return(doc); } if (fieldVal is List <object> list) { fieldVal = new DynamicJsonArray(list); } if (fieldVal is Document d2) { fieldVal = d2.Data; } result[key] = fieldVal; } } if (doc == null) { doc = new Document { Id = _context.GetLazyString(lowerId) }; } return(ReturnProjection(result, doc, scoreDoc, _context)); } }
public abstract (Document Document, List <Document> List) Get(Lucene.Net.Documents.Document input, Lucene.Net.Search.ScoreDoc scoreDoc, IState state, CancellationToken token);