public virtual void TestComplexNested()
{
PayloadNearQuery query;
TopDocs hits;
// combine ordered and unordered spans with some nesting to make sure all payloads are counted
SpanQuery q1 = NewPhraseQuery("field", "nine hundred", true);
SpanQuery q2 = NewPhraseQuery("field", "ninety nine", true);
SpanQuery q3 = NewPhraseQuery("field", "nine ninety", false);
SpanQuery q4 = NewPhraseQuery("field", "hundred nine", false);
SpanQuery[] clauses = new SpanQuery[] { new PayloadNearQuery(new SpanQuery[] { q1, q2 }, 0, true), new PayloadNearQuery(new SpanQuery[] { q3, q4 }, 0, false) };
query = new PayloadNearQuery(clauses, 0, false);
hits = searcher.Search(query, null, 100);
Assert.IsTrue(hits != null, "hits is null and it shouldn't be");
// should be only 1 hit - doc 999
Assert.IsTrue(hits.ScoreDocs.Length == 1, "should only be one hit");
// the score should be 3 - the average of all the underlying payloads
ScoreDoc doc = hits.ScoreDocs[0];
// System.out.println("Doc: " + doc.toString());
// System.out.println("Explain: " + searcher.explain(query, doc.doc));
Assert.IsTrue(doc.Score == 3, doc.Score + " does not equal: " + 3);
}
public virtual void Test()
{
BoostingTermQuery query = new BoostingTermQuery(new Term("field", "seventy"));
TopDocs hits = searcher.Search(query, null, 100);
Assert.IsTrue(hits != null, "hits is null and it shouldn't be");
Assert.IsTrue(hits.TotalHits == 100, "hits Size: " + hits.TotalHits + " is not: " + 100);
//they should all have the exact same score, because they all contain seventy once, and we set
//all the other similarity factors to be 1
Assert.IsTrue(hits.GetMaxScore() == 1, hits.GetMaxScore() + " does not equal: " + 1);
for (int i = 0; i < hits.ScoreDocs.Length; i++)
{
ScoreDoc doc = hits.ScoreDocs[i];
Assert.IsTrue(doc.score == 1, doc.score + " does not equal: " + 1);
}
CheckHits.CheckExplanations(query, PayloadHelper.FIELD, searcher, true);
Lucene.Net.Search.Spans.Spans spans = query.GetSpans(searcher.GetIndexReader());
Assert.IsTrue(spans != null, "spans is null and it shouldn't be");
Assert.IsTrue(spans is TermSpans, "spans is not an instanceof " + typeof(TermSpans));
/*float score = hits.score(0);
* for (int i =1; i < hits.length(); i++)
* {
* Assert.IsTrue(score == hits.score(i), "scores are not equal and they should be");
* }*/
}
protected Document ReturnProjection(DynamicJsonValue result, Document doc, Lucene.Net.Search.ScoreDoc scoreDoc, JsonOperationContext context)
{
result[Constants.Documents.Metadata.Key] = new DynamicJsonValue
{
[Constants.Documents.Metadata.Projection] = true
};
var newData = context.ReadObject(result, "projection result");
try
{
if (ReferenceEquals(newData, doc.Data) == false)
{
doc.Data?.Dispose();
}
}
catch (Exception)
{
newData.Dispose();
throw;
}
doc.Data = newData;
FinishDocumentSetup(doc, scoreDoc);
return(doc);
}
public virtual void Test()
{
PayloadNearQuery query;
TopDocs hits;
query = NewPhraseQuery("field", "twenty two", true);
QueryUtils.Check(query);
// all 10 hits should have score = 3 because adjacent terms have payloads of 2,4
// and all the similarity factors are set to 1
hits = searcher.Search(query, null, 100);
Assert.IsTrue(hits != null, "hits is null and it shouldn't be");
Assert.IsTrue(hits.TotalHits == 10, "should be 10 hits");
for (int j = 0; j < hits.ScoreDocs.Length; j++)
{
ScoreDoc doc = hits.ScoreDocs[j];
Assert.IsTrue(doc.Score == 3, doc.Score + " does not equal: " + 3);
}
for (int i = 1; i < 10; i++)
{
query = NewPhraseQuery("field", English.IntToEnglish(i) + " hundred", true);
// all should have score = 3 because adjacent terms have payloads of 2,4
// and all the similarity factors are set to 1
hits = searcher.Search(query, null, 100);
Assert.IsTrue(hits != null, "hits is null and it shouldn't be");
Assert.IsTrue(hits.TotalHits == 100, "should be 100 hits");
for (int j = 0; j < hits.ScoreDocs.Length; j++)
{
ScoreDoc doc = hits.ScoreDocs[j];
// System.out.println("Doc: " + doc.toString());
// System.out.println("Explain: " + searcher.explain(query, doc.doc));
Assert.IsTrue(doc.Score == 3, doc.Score + " does not equal: " + 3);
}
}
}
protected void FinishDocumentSetup(Document doc, Lucene.Net.Search.ScoreDoc scoreDoc)
{
if (doc == null || scoreDoc == null)
{
return;
}
doc.IndexScore = scoreDoc.Score;
if (_query?.Distances != null)
{
doc.Distance = _query.Distances.Get(scoreDoc.Doc);
}
}
public virtual void TestIgnoreSpanScorer()
{
PayloadTermQuery query = new PayloadTermQuery(new Term(PayloadHelper.MULTI_FIELD, "seventy"), new MaxPayloadFunction(), false);
IndexSearcher theSearcher = new IndexSearcher(directory, true, null);
theSearcher.Similarity = new FullSimilarity();
TopDocs hits = searcher.Search(query, null, 100, null);
Assert.IsTrue(hits != null, "hits is null and it shouldn't be");
Assert.IsTrue(hits.TotalHits == 100, "hits Size: " + hits.TotalHits + " is not: " + 100);
//they should all have the exact same score, because they all contain seventy once, and we set
//all the other similarity factors to be 1
//System.out.println("Hash: " + seventyHash + " Twice Hash: " + 2*seventyHash);
Assert.IsTrue(hits.MaxScore == 4.0, hits.MaxScore + " does not equal: " + 4.0);
//there should be exactly 10 items that score a 4, all the rest should score a 2
//The 10 items are: 70 + i*100 where i in [0-9]
int numTens = 0;
for (int i = 0; i < hits.ScoreDocs.Length; i++)
{
ScoreDoc doc = hits.ScoreDocs[i];
if (doc.Doc % 10 == 0)
{
numTens++;
Assert.IsTrue(doc.Score == 4.0, doc.Score + " does not equal: " + 4.0);
}
else
{
Assert.IsTrue(doc.Score == 2, doc.Score + " does not equal: " + 2);
}
}
Assert.IsTrue(numTens == 10, numTens + " does not equal: " + 10);
CheckHits.CheckExplanations(query, "field", searcher, true);
Lucene.Net.Search.Spans.Spans spans = query.GetSpans(searcher.IndexReader, null);
Assert.IsTrue(spans != null, "spans is null and it shouldn't be");
Assert.IsTrue(spans is TermSpans, "spans is not an instanceof " + typeof(TermSpans));
//should be two matches per document
int count = 0;
//100 hits times 2 matches per hit, we should have 200 in count
while (spans.Next(null))
{
count++;
}
}
public virtual void TestLongerSpan()
{
PayloadNearQuery query;
TopDocs hits;
query = NewPhraseQuery("field", "nine hundred ninety nine", true);
hits = searcher.Search(query, null, 100);
ScoreDoc doc = hits.ScoreDocs[0];
// System.out.println("Doc: " + doc.toString());
// System.out.println("Explain: " + searcher.explain(query, doc.doc));
Assert.IsTrue(hits != null, "hits is null and it shouldn't be");
Assert.IsTrue(hits.TotalHits == 1, "there should only be one hit");
// should have score = 3 because adjacent terms have payloads of 2,4
Assert.IsTrue(doc.Score == 3, doc.Score + " does not equal: " + 3);
}
public void Query(string searchText)
{
//state the file location of the index
DirectoryInfo directoryInfo = new DirectoryInfo(_Location);
Lucene.Net.Store.Directory dir = Lucene.Net.Store.FSDirectory.Open(directoryInfo);
//create an index searcher that will perform the search
Lucene.Net.Search.IndexSearcher searcher = new Lucene.Net.Search.IndexSearcher(dir, true);
//build a query object
Lucene.Net.Index.Term searchTerm = new Lucene.Net.Index.Term("content", searchText);
Lucene.Net.Search.Query query = new Lucene.Net.Search.TermQuery(searchTerm);
//execute the query
Lucene.Net.Search.TopDocs hits = searcher.Search(query, null, 100);
//iterate over the results.
for (int i = 0; i < hits.TotalHits; i++)
{
Lucene.Net.Search.ScoreDoc doc = hits.ScoreDocs[i];
}
}
public override Document Get(Lucene.Net.Documents.Document input, Lucene.Net.Search.ScoreDoc scoreDoc, IState state)
{
using (RetrieverScope?.Start())
{
if (TryGetKey(input, state, out string id) == false)
{
throw new InvalidOperationException($"Could not extract '{Constants.Documents.Indexing.Fields.DocumentIdFieldName}' from index.");
}
if (FieldsToFetch.IsProjection)
{
return(GetProjection(input, scoreDoc, id, state));
}
using (_storageScope = _storageScope?.Start() ?? RetrieverScope?.For(nameof(QueryTimingsScope.Names.Storage)))
{
var doc = DirectGet(null, id, DocumentFields, state);
FinishDocumentSetup(doc, scoreDoc);
return(doc);
}
}
}
protected Document AddProjectionToResult(Document doc, Lucene.Net.Search.ScoreDoc scoreDoc, FieldsToFetch fieldsToFetch, DynamicJsonValue result, string key, object fieldVal)
{
if (fieldsToFetch.SingleBodyOrMethodWithNoAlias)
{
Document newDoc = null;
if (fieldVal is BlittableJsonReaderObject nested)
{
newDoc = new Document
{
Id = doc.Id,
ChangeVector = doc.ChangeVector,
Data = nested,
Etag = doc.Etag,
Flags = doc.Flags,
LastModified = doc.LastModified,
LowerId = doc.LowerId,
NonPersistentFlags = doc.NonPersistentFlags,
StorageId = doc.StorageId,
TransactionMarker = doc.TransactionMarker
};
}
else if (fieldVal is Document d)
{
newDoc = d;
}
else
{
ThrowInvalidQueryBodyResponse(fieldVal);
}
FinishDocumentSetup(newDoc, scoreDoc);
return(newDoc);
}
AddProjectionToResult(result, key, fieldVal);
return(null);
}
public override (Document Document, List <Document> List) Get(Lucene.Net.Documents.Document input, Lucene.Net.Search.ScoreDoc scoreDoc, IState state, CancellationToken token)
{
throw new NotSupportedException("Graph Queries do not deal with Lucene indexes.");
}
private Document GetProjectionFromDocumentInternal(Document doc, Lucene.Net.Documents.Document luceneDoc, Lucene.Net.Search.ScoreDoc scoreDoc, FieldsToFetch fieldsToFetch, JsonOperationContext context, IState state)
{
var result = new DynamicJsonValue();
foreach (var fieldToFetch in fieldsToFetch.Fields.Values)
{
if (TryGetValue(fieldToFetch, doc, luceneDoc, state, fieldsToFetch.IndexFields, fieldsToFetch.AnyDynamicIndexFields, out var key, out var fieldVal) == false &&
fieldToFetch.QueryField != null && fieldToFetch.QueryField.HasSourceAlias)
{
continue;
}
var immediateResult = AddProjectionToResult(doc, scoreDoc, fieldsToFetch, result, key, fieldVal);
if (immediateResult != null)
{
return(immediateResult);
}
}
return(ReturnProjection(result, doc, scoreDoc, context));
}
public Document GetProjectionFromDocument(Document doc, Lucene.Net.Documents.Document luceneDoc, Lucene.Net.Search.ScoreDoc scoreDoc, FieldsToFetch fieldsToFetch, JsonOperationContext context, IState state)
{
using (RetrieverScope?.Start())
using (_projectionScope = _projectionScope?.Start() ?? RetrieverScope?.For(nameof(QueryTimingsScope.Names.Projection)))
{
return(GetProjectionFromDocumentInternal(doc, luceneDoc, scoreDoc, fieldsToFetch, context, state));
}
}
protected (Document Document, List <Document> List) GetProjection(Lucene.Net.Documents.Document input, Lucene.Net.Search.ScoreDoc scoreDoc, string lowerId, IState state, CancellationToken token)
{
using (_projectionScope = _projectionScope?.Start() ?? RetrieverScope?.For(nameof(QueryTimingsScope.Names.Projection)))
{
Document doc = null;
if (FieldsToFetch.AnyExtractableFromIndex == false)
{
using (_projectionStorageScope = _projectionStorageScope?.Start() ?? _projectionScope?.For(nameof(QueryTimingsScope.Names.Storage)))
doc = DirectGet(input, lowerId, DocumentFields.All, state);
if (doc == null)
{
if (FieldsToFetch.Projection.MustExtractFromDocument)
{
if (FieldsToFetch.Projection.MustExtractOrThrow)
{
FieldsToFetch.Projection.ThrowCouldNotExtractProjectionOnDocumentBecauseDocumentDoesNotExistException(lowerId);
}
}
return(default);
public abstract Document Get(Lucene.Net.Documents.Document input, Lucene.Net.Search.ScoreDoc scoreDoc, IState state);
public override Document Get(Lucene.Net.Documents.Document input, Lucene.Net.Search.ScoreDoc scoreDoc, IState state)
{
throw new NotSupportedException("Graph Queries do not deal with Lucene indexes.");
}
private void TestHits(ScoreDoc[] hits, int expectedCount, IndexReader reader)
{
int hitCount = hits.Length;
Assert.AreEqual(expectedCount, hitCount, "wrong number of hits");
for (int i = 0; i < hitCount; i++)
{
reader.Document(hits[i].doc);
reader.GetTermFreqVectors(hits[i].doc);
}
}
protected Document GetProjection(Lucene.Net.Documents.Document input, Lucene.Net.Search.ScoreDoc scoreDoc, string lowerId, IState state)
{
using (_projectionScope = _projectionScope?.Start() ?? RetrieverScope?.For(nameof(QueryTimingsScope.Names.Projection)))
{
Document doc = null;
if (FieldsToFetch.AnyExtractableFromIndex == false)
{
using (_projectionStorageScope = _projectionStorageScope?.Start() ?? _projectionScope?.For(nameof(QueryTimingsScope.Names.Storage)))
doc = DirectGet(input, lowerId, DocumentFields.All, state);
if (doc == null)
{
return(null);
}
return(GetProjectionFromDocumentInternal(doc, input, scoreDoc, FieldsToFetch, _context, state));
}
var documentLoaded = false;
var result = new DynamicJsonValue();
Dictionary <string, FieldsToFetch.FieldToFetch> fields = null;
if (FieldsToFetch.ExtractAllFromIndex)
{
fields = input.GetFields()
.Where(x => x.Name != Constants.Documents.Indexing.Fields.DocumentIdFieldName &&
x.Name != Constants.Documents.Indexing.Fields.SourceDocumentIdFieldName &&
x.Name != Constants.Documents.Indexing.Fields.ReduceKeyHashFieldName &&
x.Name != Constants.Documents.Indexing.Fields.ReduceKeyValueFieldName &&
x.Name != Constants.Documents.Indexing.Fields.ValueFieldName &&
FieldUtil.GetRangeTypeFromFieldName(x.Name) == RangeType.None)
.Distinct(UniqueFieldNames.Instance)
.ToDictionary(x => x.Name, x => new FieldsToFetch.FieldToFetch(x.Name, null, null, x.IsStored, isDocumentId: false, isTimeSeries: false));
}
if (fields == null)
{
fields = FieldsToFetch.Fields;
}
else if (FieldsToFetch.Fields != null && FieldsToFetch.Fields.Count > 0)
{
foreach (var kvp in FieldsToFetch.Fields)
{
if (fields.ContainsKey(kvp.Key))
{
continue;
}
fields[kvp.Key] = kvp.Value;
}
}
foreach (var fieldToFetch in fields.Values)
{
if (TryExtractValueFromIndex(fieldToFetch, input, result, state))
{
continue;
}
if (documentLoaded == false)
{
using (_projectionStorageScope = _projectionStorageScope?.Start() ?? _projectionScope?.For(nameof(QueryTimingsScope.Names.Storage)))
doc = DirectGet(input, lowerId, DocumentFields.All, state);
documentLoaded = true;
}
if (doc == null)
{
continue;
}
if (TryGetValue(fieldToFetch, doc, input, state, FieldsToFetch.IndexFields, FieldsToFetch.AnyDynamicIndexFields, out var key, out var fieldVal))
{
if (FieldsToFetch.SingleBodyOrMethodWithNoAlias)
{
if (fieldVal is BlittableJsonReaderObject nested)
{
doc.Data = nested;
}
else if (fieldVal is Document d)
{
doc = d;
}
else
{
ThrowInvalidQueryBodyResponse(fieldVal);
}
FinishDocumentSetup(doc, scoreDoc);
return(doc);
}
if (fieldVal is List <object> list)
{
fieldVal = new DynamicJsonArray(list);
}
if (fieldVal is Document d2)
{
fieldVal = d2.Data;
}
result[key] = fieldVal;
}
}
if (doc == null)
{
doc = new Document
{
Id = _context.GetLazyString(lowerId)
};
}
return(ReturnProjection(result, doc, scoreDoc, _context));
}
}
public abstract (Document Document, List <Document> List) Get(Lucene.Net.Documents.Document input, Lucene.Net.Search.ScoreDoc scoreDoc, IState state, CancellationToken token);
