public virtual void TestNestedSpans()
{
SpanTermQuery stq;
Spans spans;
IndexSearcher searcher = GetSearcher();
stq = new SpanTermQuery(new Term(PayloadHelper.FIELD, "mark"));
spans = stq.GetSpans(searcher.GetIndexReader());
Assert.IsTrue(spans != null, "spans is null and it shouldn't be");
CheckSpans(spans, 0, null);
SpanQuery[] clauses = new SpanQuery[3];
clauses[0] = new SpanTermQuery(new Term(PayloadHelper.FIELD, "rr"));
clauses[1] = new SpanTermQuery(new Term(PayloadHelper.FIELD, "yy"));
clauses[2] = new SpanTermQuery(new Term(PayloadHelper.FIELD, "xx"));
SpanNearQuery spanNearQuery = new SpanNearQuery(clauses, 12, false);
spans = spanNearQuery.GetSpans(searcher.GetIndexReader());
Assert.IsTrue(spans != null, "spans is null and it shouldn't be");
CheckSpans(spans, 2, new int[] { 3, 3 });
clauses[0] = new SpanTermQuery(new Term(PayloadHelper.FIELD, "xx"));
clauses[1] = new SpanTermQuery(new Term(PayloadHelper.FIELD, "rr"));
clauses[2] = new SpanTermQuery(new Term(PayloadHelper.FIELD, "yy"));
spanNearQuery = new SpanNearQuery(clauses, 6, true);
spans = spanNearQuery.GetSpans(searcher.GetIndexReader());
Assert.IsTrue(spans != null, "spans is null and it shouldn't be");
CheckSpans(spans, 1, new int[] { 3 });
clauses = new SpanQuery[2];
clauses[0] = new SpanTermQuery(new Term(PayloadHelper.FIELD, "xx"));
clauses[1] = new SpanTermQuery(new Term(PayloadHelper.FIELD, "rr"));
spanNearQuery = new SpanNearQuery(clauses, 6, true);
// xx within 6 of rr
SpanQuery[] clauses2 = new SpanQuery[2];
clauses2[0] = new SpanTermQuery(new Term(PayloadHelper.FIELD, "yy"));
clauses2[1] = spanNearQuery;
SpanNearQuery nestedSpanNearQuery = new SpanNearQuery(clauses2, 6, false);
// yy within 6 of xx within 6 of rr
spans = nestedSpanNearQuery.GetSpans(searcher.GetIndexReader());
Assert.IsTrue(spans != null, "spans is null and it shouldn't be");
CheckSpans(spans, 2, new int[] { 3, 3 });
}
public virtual void TestNearSpansNext()
{
SpanNearQuery q = MakeQuery();
Spans span = q.GetSpans(searcher.GetIndexReader());
Assert.AreEqual(true, span.Next());
Assert.AreEqual(S(0, 0, 3), S(span));
Assert.AreEqual(true, span.Next());
Assert.AreEqual(S(1, 0, 4), S(span));
Assert.AreEqual(false, span.Next());
}
public override void SetUp()
{
base.SetUp();
PayloadHelper helper = new PayloadHelper();
searcher = helper.SetUp(similarity, 1000);
indexReader = searcher.GetIndexReader();
}
public virtual void TestHeavilyNestedSpanQuery()
{
Spans spans;
IndexSearcher searcher = GetSearcher();
SpanQuery[] clauses = new SpanQuery[3];
clauses[0] = new SpanTermQuery(new Term(PayloadHelper.FIELD, "one"));
clauses[1] = new SpanTermQuery(new Term(PayloadHelper.FIELD, "two"));
clauses[2] = new SpanTermQuery(new Term(PayloadHelper.FIELD, "three"));
SpanNearQuery spanNearQuery = new SpanNearQuery(clauses, 5, true);
clauses = new SpanQuery[3];
clauses[0] = spanNearQuery;
clauses[1] = new SpanTermQuery(new Term(PayloadHelper.FIELD, "five"));
clauses[2] = new SpanTermQuery(new Term(PayloadHelper.FIELD, "six"));
SpanNearQuery spanNearQuery2 = new SpanNearQuery(clauses, 6, true);
SpanQuery[] clauses2 = new SpanQuery[2];
clauses2[0] = new SpanTermQuery(new Term(PayloadHelper.FIELD, "eleven"));
clauses2[1] = new SpanTermQuery(new Term(PayloadHelper.FIELD, "ten"));
SpanNearQuery spanNearQuery3 = new SpanNearQuery(clauses2, 2, false);
SpanQuery[] clauses3 = new SpanQuery[3];
clauses3[0] = new SpanTermQuery(new Term(PayloadHelper.FIELD, "nine"));
clauses3[1] = spanNearQuery2;
clauses3[2] = spanNearQuery3;
SpanNearQuery nestedSpanNearQuery = new SpanNearQuery(clauses3, 6, false);
spans = nestedSpanNearQuery.GetSpans(searcher.GetIndexReader());
Assert.IsTrue(spans != null, "spans is null and it shouldn't be");
CheckSpans(spans, 2, new int[] { 8, 8 });
}
public virtual void TestFirstClauseWithoutPayload()
{
Spans spans;
IndexSearcher searcher = GetSearcher();
SpanQuery[] clauses = new SpanQuery[3];
clauses[0] = new SpanTermQuery(new Term(PayloadHelper.FIELD, "nopayload"));
clauses[1] = new SpanTermQuery(new Term(PayloadHelper.FIELD, "qq"));
clauses[2] = new SpanTermQuery(new Term(PayloadHelper.FIELD, "ss"));
SpanNearQuery spanNearQuery = new SpanNearQuery(clauses, 6, true);
SpanQuery[] clauses2 = new SpanQuery[2];
clauses2[0] = new SpanTermQuery(new Term(PayloadHelper.FIELD, "pp"));
clauses2[1] = spanNearQuery;
SpanNearQuery snq = new SpanNearQuery(clauses2, 6, false);
SpanQuery[] clauses3 = new SpanQuery[2];
clauses3[0] = new SpanTermQuery(new Term(PayloadHelper.FIELD, "np"));
clauses3[1] = snq;
SpanNearQuery nestedSpanNearQuery = new SpanNearQuery(clauses3, 6, false);
spans = nestedSpanNearQuery.GetSpans(searcher.GetIndexReader());
Assert.IsTrue(spans != null, "spans is null and it shouldn't be");
CheckSpans(spans, 1, new int[] { 3 });
}
public virtual void TestPayloadSpanUtil()
{
RAMDirectory directory = new RAMDirectory();
PayloadAnalyzer analyzer = new PayloadAnalyzer(this);
IndexWriter writer = new IndexWriter(directory, analyzer, true);
writer.SetSimilarity(similarity);
Document doc = new Document();
doc.Add(new Field(PayloadHelper.FIELD, "xx rr yy mm pp", Field.Store.YES, Field.Index.ANALYZED));
writer.AddDocument(doc);
writer.Close();
IndexSearcher searcher = new IndexSearcher(directory);
IndexReader reader = searcher.GetIndexReader();
PayloadSpanUtil psu = new PayloadSpanUtil(reader);
System.Collections.Generic.ICollection <byte[]> payloads = psu.GetPayloadsForQuery(new TermQuery(new Term(PayloadHelper.FIELD, "rr")));
if (DEBUG)
{
System.Console.Out.WriteLine("Num payloads:" + payloads.Count);
}
System.Collections.IEnumerator it = payloads.GetEnumerator();
while (it.MoveNext())
{
byte[] bytes = (byte[])it.Current;
if (DEBUG)
{
System.Console.Out.WriteLine(new System.String(System.Text.UTF8Encoding.UTF8.GetChars(bytes)));
}
}
}
// Test that queries based on reverse/ordFieldScore returns docs with expected score.
private void DoTestExactScore(System.String field, bool inOrder)
{
IndexSearcher s = new IndexSearcher(dir);
ValueSource vs;
if (inOrder)
{
vs = new OrdFieldSource(field);
}
else
{
vs = new ReverseOrdFieldSource(field);
}
Query q = new ValueSourceQuery(vs);
TopDocs td = s.Search(q, null, 1000);
Assert.AreEqual(N_DOCS, td.totalHits, "All docs should be matched!");
ScoreDoc[] sd = td.scoreDocs;
for (int i = 0; i < sd.Length; i++)
{
float score = sd[i].score;
System.String id = s.GetIndexReader().Document(sd[i].doc).Get(ID_FIELD);
Log("-------- " + i + ". Explain doc " + id);
Log(s.Explain(q, sd[i].doc));
float expectedScore = N_DOCS - i;
Assert.AreEqual(expectedScore, score, TEST_SCORE_TOLERANCE_DELTA, "score of result " + i + " shuould be " + expectedScore + " != " + score);
System.String expectedId = inOrder?Id2String(N_DOCS - i):Id2String(i + 1); // reverse ==> smaller values first
Assert.IsTrue(expectedId.Equals(id), "id of result " + i + " shuould be " + expectedId + " != " + score);
}
}
public override void SetUp()
{
base.SetUp();
PayloadHelper helper = new PayloadHelper();
searcher = helper.SetUp(similarity, 1000);
indexReader = searcher.GetIndexReader();
}
public virtual void TestSpanNearOrderedOverlap()
{
bool ordered = true;
int slop = 1;
SpanNearQuery snq = new SpanNearQuery(new SpanQuery[] { MakeSpanTermQuery("t1"), MakeSpanTermQuery("t2"), MakeSpanTermQuery("t3") }, slop, ordered);
Spans spans = snq.GetSpans(searcher.GetIndexReader());
Assert.IsTrue(spans.Next(), "first range");
Assert.AreEqual(11, spans.Doc(), "first doc");
Assert.AreEqual(0, spans.Start(), "first start");
Assert.AreEqual(4, spans.End(), "first end");
Assert.IsTrue(spans.Next(), "second range");
Assert.AreEqual(11, spans.Doc(), "second doc");
Assert.AreEqual(2, spans.Start(), "second start");
Assert.AreEqual(6, spans.End(), "second end");
Assert.IsFalse(spans.Next(), "third range");
}
public virtual void TestSpansSkipTo()
{
SpanTermQuery t1 = new SpanTermQuery(new Term("field", "seventy"));
SpanTermQuery t2 = new SpanTermQuery(new Term("field", "seventy"));
Spans s1 = t1.GetSpans(searcher.GetIndexReader());
Spans s2 = t2.GetSpans(searcher.GetIndexReader());
Assert.IsTrue(s1.Next());
Assert.IsTrue(s2.Next());
bool hasMore = true;
do
{
hasMore = SkipToAccoringToJavaDocs(s1, s1.Doc());
Assert.AreEqual(hasMore, s2.SkipTo(s2.Doc()));
Assert.AreEqual(s1.Doc(), s2.Doc());
}while (hasMore);
}
public virtual void TestSpans0()
{
SpanQuery q1 = new SpanTermQuery(new Term("gender", "female"));
SpanQuery q2 = new SpanTermQuery(new Term("first", "james"));
SpanQuery q = new SpanOrQuery(new SpanQuery[] { q1, new FieldMaskingSpanQuery(q2, "gender") });
Check(q, new int[] { 0, 1, 2, 3, 4 });
Spans span = q.GetSpans(searcher.GetIndexReader());
Assert.AreEqual(true, span.Next());
Assert.AreEqual(S(0, 0, 1), S(span));
Assert.AreEqual(true, span.Next());
Assert.AreEqual(S(1, 0, 1), S(span));
Assert.AreEqual(true, span.Next());
Assert.AreEqual(S(1, 1, 2), S(span));
Assert.AreEqual(true, span.Next());
Assert.AreEqual(S(2, 0, 1), S(span));
Assert.AreEqual(true, span.Next());
Assert.AreEqual(S(2, 1, 2), S(span));
Assert.AreEqual(true, span.Next());
Assert.AreEqual(S(2, 2, 3), S(span));
Assert.AreEqual(true, span.Next());
Assert.AreEqual(S(3, 0, 1), S(span));
Assert.AreEqual(true, span.Next());
Assert.AreEqual(S(4, 0, 1), S(span));
Assert.AreEqual(true, span.Next());
Assert.AreEqual(S(4, 1, 2), S(span));
Assert.AreEqual(false, span.Next());
}
/* Open pre-lockless index, add docs, do a delete &
* setNorm, and search */
public virtual void ChangeIndexNoAdds(System.String dirName, bool autoCommit)
{
dirName = FullDir(dirName);
Directory dir = FSDirectory.Open(new System.IO.FileInfo(dirName));
// make sure searching sees right # hits
IndexSearcher searcher = new IndexSearcher(dir);
ScoreDoc[] hits = searcher.Search(new TermQuery(new Term("content", "aaa")), null, 1000).ScoreDocs;
Assert.AreEqual(34, hits.Length, "wrong number of hits");
Document d = searcher.Doc(hits[0].doc);
Assert.AreEqual("21", d.Get("id"), "wrong first document");
searcher.Close();
// make sure we can do a delete & setNorm against this
// pre-lockless segment:
IndexReader reader = IndexReader.Open(dir);
Term searchTerm = new Term("id", "6");
int delCount = reader.DeleteDocuments(searchTerm);
Assert.AreEqual(1, delCount, "wrong delete count");
reader.SetNorm(22, "content", (float)2.0);
reader.Close();
// make sure they "took":
searcher = new IndexSearcher(dir);
hits = searcher.Search(new TermQuery(new Term("content", "aaa")), null, 1000).ScoreDocs;
Assert.AreEqual(33, hits.Length, "wrong number of hits");
d = searcher.Doc(hits[0].doc);
Assert.AreEqual("22", d.Get("id"), "wrong first document");
TestHits(hits, 33, searcher.GetIndexReader());
searcher.Close();
// optimize
IndexWriter writer = new IndexWriter(dir, autoCommit, new WhitespaceAnalyzer(), false);
writer.Optimize();
writer.Close();
searcher = new IndexSearcher(dir);
hits = searcher.Search(new TermQuery(new Term("content", "aaa")), null, 1000).ScoreDocs;
Assert.AreEqual(33, hits.Length, "wrong number of hits");
d = searcher.Doc(hits[0].doc);
Assert.AreEqual("22", d.Get("id"), "wrong first document");
TestHits(hits, 33, searcher.GetIndexReader());
searcher.Close();
dir.Close();
}
// Test that FieldScoreQuery returns docs with expected score.
private void DoTestExactScore(System.String field, FieldScoreQuery.Type tp)
{
IndexSearcher s = new IndexSearcher(dir);
Query q = new FieldScoreQuery(field, tp);
TopDocs td = s.Search(q, null, 1000);
Assert.AreEqual(N_DOCS, td.totalHits, "All docs should be matched!");
ScoreDoc[] sd = td.scoreDocs;
for (int i = 0; i < sd.Length; i++)
{
float score = sd[i].score;
Log(s.Explain(q, sd[i].doc));
System.String id = s.GetIndexReader().Document(sd[i].doc).Get(ID_FIELD);
float expectedScore = ExpectedFieldScore(id); // "ID7" --> 7.0
Assert.AreEqual(expectedScore, score, TEST_SCORE_TOLERANCE_DELTA, "score of " + id + " shuould be " + expectedScore + " != " + score);
}
}
public virtual void TestShrinkToAfterShortestMatch3()
{
RAMDirectory directory = new RAMDirectory();
IndexWriter writer = new IndexWriter(directory, new TestPayloadAnalyzer(this), IndexWriter.MaxFieldLength.LIMITED);
Document doc = new Document();
doc.Add(new Field("content", new System.IO.StreamReader(new System.IO.MemoryStream(System.Text.Encoding.ASCII.GetBytes("j k a l f k k p a t a k l k t a")))));
writer.AddDocument(doc);
writer.Close();
IndexSearcher is_Renamed = new IndexSearcher(directory);
SpanTermQuery stq1 = new SpanTermQuery(new Term("content", "a"));
SpanTermQuery stq2 = new SpanTermQuery(new Term("content", "k"));
SpanQuery[] sqs = new SpanQuery[] { stq1, stq2 };
SpanNearQuery snq = new SpanNearQuery(sqs, 0, true);
Spans spans = snq.GetSpans(is_Renamed.GetIndexReader());
TopDocs topDocs = is_Renamed.Search(snq, 1);
System.Collections.Hashtable payloadSet = new System.Collections.Hashtable();
for (int i = 0; i < topDocs.scoreDocs.Length; i++)
{
while (spans.Next())
{
System.Collections.Generic.ICollection <byte[]> payloads = spans.GetPayload();
for (System.Collections.IEnumerator it = payloads.GetEnumerator(); it.MoveNext();)
{
SupportClass.CollectionsHelper.AddIfNotContains(payloadSet, new System.String(System.Text.UTF8Encoding.UTF8.GetChars((byte[])it.Current)));
}
}
}
Assert.AreEqual(2, payloadSet.Count);
if (DEBUG)
{
System.Collections.IEnumerator pit = payloadSet.GetEnumerator();
while (pit.MoveNext())
{
System.Console.Out.WriteLine("match:" + pit.Current);
}
}
Assert.IsTrue(payloadSet.Contains("a:Noise:10"));
Assert.IsTrue(payloadSet.Contains("k:Noise:11"));
}
public virtual void searchIndex(System.String dirName, System.String oldName)
{
//QueryParser parser = new QueryParser("contents", new WhitespaceAnalyzer());
//Query query = parser.parse("handle:1");
dirName = FullDir(dirName);
Directory dir = FSDirectory.Open(new System.IO.FileInfo(dirName));
IndexSearcher searcher = new IndexSearcher(dir);
IndexReader reader = searcher.GetIndexReader();
_TestUtil.CheckIndex(dir);
for (int i = 0; i < 35; i++)
{
if (!reader.IsDeleted(i))
{
Document d = reader.Document(i);
System.Collections.IList fields = d.GetFields();
if (!oldName.StartsWith("19.") && !oldName.StartsWith("20.") && !oldName.StartsWith("21.") && !oldName.StartsWith("22."))
{
if (d.GetField("content3") == null)
{
Assert.AreEqual(5, fields.Count);
Field f = (Field) d.GetField("id");
Assert.AreEqual("" + i, f.StringValue());
f = (Field) d.GetField("utf8");
Assert.AreEqual("Lu\uD834\uDD1Ece\uD834\uDD60ne \u0000 \u2620 ab\ud917\udc17cd", f.StringValue());
f = (Field) d.GetField("autf8");
Assert.AreEqual("Lu\uD834\uDD1Ece\uD834\uDD60ne \u0000 \u2620 ab\ud917\udc17cd", f.StringValue());
f = (Field) d.GetField("content2");
Assert.AreEqual("here is more content with aaa aaa aaa", f.StringValue());
f = (Field) d.GetField("fie\u2C77ld");
Assert.AreEqual("field with non-ascii name", f.StringValue());
}
}
}
// Only ID 7 is deleted
else
Assert.AreEqual(7, i);
}
ScoreDoc[] hits = searcher.Search(new TermQuery(new Term("content", "aaa")), null, 1000).scoreDocs;
// First document should be #21 since it's norm was
// increased:
Document d2 = searcher.Doc(hits[0].doc);
Assert.AreEqual("21", d2.Get("id"), "didn't get the right document first");
TestHits(hits, 34, searcher.GetIndexReader());
if (!oldName.StartsWith("19.") && !oldName.StartsWith("20.") && !oldName.StartsWith("21.") && !oldName.StartsWith("22."))
{
// Test on indices >= 2.3
hits = searcher.Search(new TermQuery(new Term("utf8", "\u0000")), null, 1000).scoreDocs;
Assert.AreEqual(34, hits.Length);
hits = searcher.Search(new TermQuery(new Term("utf8", "Lu\uD834\uDD1Ece\uD834\uDD60ne")), null, 1000).scoreDocs;
Assert.AreEqual(34, hits.Length);
hits = searcher.Search(new TermQuery(new Term("utf8", "ab\ud917\udc17cd")), null, 1000).scoreDocs;
Assert.AreEqual(34, hits.Length);
}
searcher.Close();
dir.Close();
}
/* Open pre-lockless index, add docs, do a delete &
* setNorm, and search */
public virtual void ChangeIndexNoAdds(System.String dirName, bool autoCommit)
{
dirName = FullDir(dirName);
Directory dir = FSDirectory.Open(new System.IO.FileInfo(dirName));
// make sure searching sees right # hits
IndexSearcher searcher = new IndexSearcher(dir);
ScoreDoc[] hits = searcher.Search(new TermQuery(new Term("content", "aaa")), null, 1000).scoreDocs;
Assert.AreEqual(34, hits.Length, "wrong number of hits");
Document d = searcher.Doc(hits[0].doc);
Assert.AreEqual("21", d.Get("id"), "wrong first document");
searcher.Close();
// make sure we can do a delete & setNorm against this
// pre-lockless segment:
IndexReader reader = IndexReader.Open(dir);
Term searchTerm = new Term("id", "6");
int delCount = reader.DeleteDocuments(searchTerm);
Assert.AreEqual(1, delCount, "wrong delete count");
reader.SetNorm(22, "content", (float) 2.0);
reader.Close();
// make sure they "took":
searcher = new IndexSearcher(dir);
hits = searcher.Search(new TermQuery(new Term("content", "aaa")), null, 1000).scoreDocs;
Assert.AreEqual(33, hits.Length, "wrong number of hits");
d = searcher.Doc(hits[0].doc);
Assert.AreEqual("22", d.Get("id"), "wrong first document");
TestHits(hits, 33, searcher.GetIndexReader());
searcher.Close();
// optimize
IndexWriter writer = new IndexWriter(dir, autoCommit, new WhitespaceAnalyzer(), false);
writer.Optimize();
writer.Close();
searcher = new IndexSearcher(dir);
hits = searcher.Search(new TermQuery(new Term("content", "aaa")), null, 1000).scoreDocs;
Assert.AreEqual(33, hits.Length, "wrong number of hits");
d = searcher.Doc(hits[0].doc);
Assert.AreEqual("22", d.Get("id"), "wrong first document");
TestHits(hits, 33, searcher.GetIndexReader());
searcher.Close();
dir.Close();
}
/* Open pre-lockless index, add docs, do a delete &
* setNorm, and search */
public virtual void ChangeIndexWithAdds(System.String dirName, bool autoCommit)
{
System.String origDirName = dirName;
dirName = FullDir(dirName);
Directory dir = FSDirectory.Open(new System.IO.FileInfo(dirName));
// open writer
IndexWriter writer = new IndexWriter(dir, autoCommit, new WhitespaceAnalyzer(), false);
// add 10 docs
for (int i = 0; i < 10; i++)
{
AddDoc(writer, 35 + i);
}
// make sure writer sees right total -- writer seems not to know about deletes in .del?
int expected;
if (Compare(origDirName, "24") < 0)
{
expected = 45;
}
else
{
expected = 46;
}
Assert.AreEqual(expected, writer.DocCount(), "wrong doc count");
writer.Close();
// make sure searching sees right # hits
IndexSearcher searcher = new IndexSearcher(dir);
ScoreDoc[] hits = searcher.Search(new TermQuery(new Term("content", "aaa")), null, 1000).scoreDocs;
Document d = searcher.Doc(hits[0].doc);
Assert.AreEqual("21", d.Get("id"), "wrong first document");
TestHits(hits, 44, searcher.GetIndexReader());
searcher.Close();
// make sure we can do delete & setNorm against this
// pre-lockless segment:
IndexReader reader = IndexReader.Open(dir);
Term searchTerm = new Term("id", "6");
int delCount = reader.DeleteDocuments(searchTerm);
Assert.AreEqual(1, delCount, "wrong delete count");
reader.SetNorm(22, "content", (float) 2.0);
reader.Close();
// make sure they "took":
searcher = new IndexSearcher(dir);
hits = searcher.Search(new TermQuery(new Term("content", "aaa")), null, 1000).scoreDocs;
Assert.AreEqual(43, hits.Length, "wrong number of hits");
d = searcher.Doc(hits[0].doc);
Assert.AreEqual("22", d.Get("id"), "wrong first document");
TestHits(hits, 43, searcher.GetIndexReader());
searcher.Close();
// optimize
writer = new IndexWriter(dir, autoCommit, new WhitespaceAnalyzer(), false);
writer.Optimize();
writer.Close();
searcher = new IndexSearcher(dir);
hits = searcher.Search(new TermQuery(new Term("content", "aaa")), null, 1000).scoreDocs;
Assert.AreEqual(43, hits.Length, "wrong number of hits");
d = searcher.Doc(hits[0].doc);
TestHits(hits, 43, searcher.GetIndexReader());
Assert.AreEqual("22", d.Get("id"), "wrong first document");
searcher.Close();
dir.Close();
}
public virtual void TestPayloadSpanUtil()
{
RAMDirectory directory = new RAMDirectory();
PayloadAnalyzer analyzer = new PayloadAnalyzer(this);
IndexWriter writer = new IndexWriter(directory, analyzer, true);
writer.SetSimilarity(similarity);
Document doc = new Document();
doc.Add(new Field(PayloadHelper.FIELD, "xx rr yy mm pp", Field.Store.YES, Field.Index.ANALYZED));
writer.AddDocument(doc);
writer.Close();
IndexSearcher searcher = new IndexSearcher(directory);
IndexReader reader = searcher.GetIndexReader();
PayloadSpanUtil psu = new PayloadSpanUtil(reader);
System.Collections.Generic.ICollection<byte[]> payloads = psu.GetPayloadsForQuery(new TermQuery(new Term(PayloadHelper.FIELD, "rr")));
if (DEBUG)
System.Console.Out.WriteLine("Num payloads:" + payloads.Count);
System.Collections.IEnumerator it = payloads.GetEnumerator();
while (it.MoveNext())
{
byte[] bytes = (byte[]) it.Current;
if (DEBUG)
System.Console.Out.WriteLine(new System.String(System.Text.UTF8Encoding.UTF8.GetChars(bytes)));
}
}
public virtual void TestShrinkToAfterShortestMatch3()
{
RAMDirectory directory = new RAMDirectory();
IndexWriter writer = new IndexWriter(directory, new TestPayloadAnalyzer(this), IndexWriter.MaxFieldLength.LIMITED);
Document doc = new Document();
doc.Add(new Field("content", new System.IO.StreamReader(new System.IO.MemoryStream(System.Text.Encoding.ASCII.GetBytes("j k a l f k k p a t a k l k t a")))));
writer.AddDocument(doc);
writer.Close();
IndexSearcher is_Renamed = new IndexSearcher(directory);
SpanTermQuery stq1 = new SpanTermQuery(new Term("content", "a"));
SpanTermQuery stq2 = new SpanTermQuery(new Term("content", "k"));
SpanQuery[] sqs = new SpanQuery[]{stq1, stq2};
SpanNearQuery snq = new SpanNearQuery(sqs, 0, true);
Spans spans = snq.GetSpans(is_Renamed.GetIndexReader());
TopDocs topDocs = is_Renamed.Search(snq, 1);
System.Collections.Hashtable payloadSet = new System.Collections.Hashtable();
for (int i = 0; i < topDocs.ScoreDocs.Length; i++)
{
while (spans.Next())
{
System.Collections.Generic.ICollection<byte[]> payloads = spans.GetPayload();
for (System.Collections.IEnumerator it = payloads.GetEnumerator(); it.MoveNext(); )
{
Support.CollectionsHelper.AddIfNotContains(payloadSet, new System.String(System.Text.UTF8Encoding.UTF8.GetChars((byte[]) it.Current)));
}
}
}
Assert.AreEqual(2, payloadSet.Count);
if (DEBUG)
{
System.Collections.IEnumerator pit = payloadSet.GetEnumerator();
while (pit.MoveNext())
{
System.Console.Out.WriteLine("match:" + pit.Current);
}
}
Assert.IsTrue(payloadSet.Contains("a:Noise:10"));
Assert.IsTrue(payloadSet.Contains("k:Noise:11"));
}
// Test that queries based on reverse/ordFieldScore returns docs with expected score.
private void DoTestExactScore(System.String field, bool inOrder)
{
IndexSearcher s = new IndexSearcher(dir);
ValueSource vs;
if (inOrder)
{
vs = new OrdFieldSource(field);
}
else
{
vs = new ReverseOrdFieldSource(field);
}
Query q = new ValueSourceQuery(vs);
TopDocs td = s.Search(q, null, 1000);
Assert.AreEqual(N_DOCS, td.totalHits, "All docs should be matched!");
ScoreDoc[] sd = td.scoreDocs;
for (int i = 0; i < sd.Length; i++)
{
float score = sd[i].score;
System.String id = s.GetIndexReader().Document(sd[i].doc).Get(ID_FIELD);
Log("-------- " + i + ". Explain doc " + id);
Log(s.Explain(q, sd[i].doc));
float expectedScore = N_DOCS - i;
Assert.AreEqual(expectedScore, score, TEST_SCORE_TOLERANCE_DELTA, "score of result " + i + " shuould be " + expectedScore + " != " + score);
System.String expectedId = inOrder?Id2String(N_DOCS - i):Id2String(i + 1); // reverse ==> smaller values first
Assert.IsTrue(expectedId.Equals(id), "id of result " + i + " shuould be " + expectedId + " != " + score);
}
}
// Test that values loaded for FieldScoreQuery are cached properly and consumes the proper RAM resources.
private void DoTestCaching(System.String field, bool inOrder)
{
IndexSearcher s = new IndexSearcher(dir);
System.Object innerArray = null;
bool warned = false; // print warning once
for (int i = 0; i < 10; i++)
{
ValueSource vs;
if (inOrder)
{
vs = new OrdFieldSource(field);
}
else
{
vs = new ReverseOrdFieldSource(field);
}
ValueSourceQuery q = new ValueSourceQuery(vs);
ScoreDoc[] h = s.Search(q, null, 1000).scoreDocs;
try
{
Assert.AreEqual(N_DOCS, h.Length, "All docs should be matched!");
IndexReader[] readers = s.GetIndexReader().GetSequentialSubReaders();
for (int j = 0; j < readers.Length; j++)
{
IndexReader reader = readers[j];
if (i == 0)
{
innerArray = q.valSrc_ForNUnit.GetValues(reader).GetInnerArray();
}
else
{
Log(i + ". compare: " + innerArray + " to " + q.valSrc_ForNUnit.GetValues(reader).GetInnerArray());
Assert.AreSame(innerArray, q.valSrc_ForNUnit.GetValues(reader).GetInnerArray(), "field values should be cached and reused!");
}
}
}
catch (System.NotSupportedException e)
{
if (!warned)
{
System.Console.Error.WriteLine("WARNING: " + TestName() + " cannot fully test values of " + q);
warned = true;
}
}
}
ValueSource vs2;
ValueSourceQuery q2;
ScoreDoc[] h2;
// verify that different values are loaded for a different field
System.String field2 = INT_FIELD;
Assert.IsFalse(field.Equals(field2)); // otherwise this test is meaningless.
if (inOrder)
{
vs2 = new OrdFieldSource(field2);
}
else
{
vs2 = new ReverseOrdFieldSource(field2);
}
q2 = new ValueSourceQuery(vs2);
h2 = s.Search(q2, null, 1000).scoreDocs;
Assert.AreEqual(N_DOCS, h2.Length, "All docs should be matched!");
IndexReader[] readers2 = s.GetIndexReader().GetSequentialSubReaders();
for (int j = 0; j < readers2.Length; j++)
{
IndexReader reader = readers2[j];
try
{
Log("compare (should differ): " + innerArray + " to " + q2.valSrc_ForNUnit.GetValues(reader).GetInnerArray());
Assert.AreNotSame(innerArray, q2.valSrc_ForNUnit.GetValues(reader).GetInnerArray(), "different values shuold be loaded for a different field!");
}
catch (System.NotSupportedException e)
{
if (!warned)
{
System.Console.Error.WriteLine("WARNING: " + TestName() + " cannot fully test values of " + q2);
warned = true;
}
}
}
// verify new values are reloaded (not reused) for a new reader
s = new IndexSearcher(dir);
if (inOrder)
{
vs2 = new OrdFieldSource(field);
}
else
{
vs2 = new ReverseOrdFieldSource(field);
}
q2 = new ValueSourceQuery(vs2);
h2 = s.Search(q2, null, 1000).scoreDocs;
Assert.AreEqual(N_DOCS, h2.Length, "All docs should be matched!");
readers2 = s.GetIndexReader().GetSequentialSubReaders();
for (int j = 0; j < readers2.Length; j++)
{
IndexReader reader = readers2[j];
try
{
Log("compare (should differ): " + innerArray + " to " + q2.valSrc_ForNUnit.GetValues(reader).GetInnerArray());
Assert.AreNotSame(innerArray, q2.valSrc_ForNUnit.GetValues(reader).GetInnerArray(), "cached field values should not be reused if reader as changed!");
}
catch (System.NotSupportedException e)
{
if (!warned)
{
System.Console.Error.WriteLine("WARNING: " + TestName() + " cannot fully test values of " + q2);
warned = true;
}
}
}
}
// Test that FieldScoreQuery returns docs with expected score.
private void DoTestExactScore(System.String field, FieldScoreQuery.Type tp)
{
IndexSearcher s = new IndexSearcher(dir);
Query q = new FieldScoreQuery(field, tp);
TopDocs td = s.Search(q, null, 1000);
Assert.AreEqual(N_DOCS, td.totalHits, "All docs should be matched!");
ScoreDoc[] sd = td.scoreDocs;
for (int i = 0; i < sd.Length; i++)
{
float score = sd[i].score;
Log(s.Explain(q, sd[i].doc));
System.String id = s.GetIndexReader().Document(sd[i].doc).Get(ID_FIELD);
float expectedScore = ExpectedFieldScore(id); // "ID7" --> 7.0
Assert.AreEqual(expectedScore, score, TEST_SCORE_TOLERANCE_DELTA, "score of " + id + " shuould be " + expectedScore + " != " + score);
}
}
// Test that values loaded for FieldScoreQuery are cached properly and consumes the proper RAM resources.
private void DoTestCaching(System.String field, FieldScoreQuery.Type tp)
{
// prepare expected array types for comparison
System.Collections.Hashtable expectedArrayTypes = new System.Collections.Hashtable();
expectedArrayTypes[FieldScoreQuery.Type.BYTE] = new sbyte[0];
expectedArrayTypes[FieldScoreQuery.Type.SHORT] = new short[0];
expectedArrayTypes[FieldScoreQuery.Type.INT] = new int[0];
expectedArrayTypes[FieldScoreQuery.Type.FLOAT] = new float[0];
IndexSearcher s = new IndexSearcher(dir);
System.Object[] innerArray = new Object[s.GetIndexReader().GetSequentialSubReaders().Length];
bool warned = false; // print warning once.
for (int i = 0; i < 10; i++)
{
FieldScoreQuery q = new FieldScoreQuery(field, tp);
ScoreDoc[] h = s.Search(q, null, 1000).scoreDocs;
Assert.AreEqual(N_DOCS, h.Length, "All docs should be matched!");
IndexReader[] readers = s.GetIndexReader().GetSequentialSubReaders();
for (int j = 0; j < readers.Length; j++)
{
IndexReader reader = readers[j];
try
{
if (i == 0)
{
innerArray[j] = q.valSrc_ForNUnit.GetValues(reader).GetInnerArray();
Log(i + ". compare: " + innerArray[j].GetType() + " to " + expectedArrayTypes[tp].GetType());
Assert.AreEqual(innerArray[j].GetType(), expectedArrayTypes[tp].GetType(), "field values should be cached in the correct array type!");
}
else
{
Log(i + ". compare: " + innerArray[j] + " to " + q.valSrc_ForNUnit.GetValues(reader).GetInnerArray());
Assert.AreSame(innerArray[j], q.valSrc_ForNUnit.GetValues(reader).GetInnerArray(), "field values should be cached and reused!");
}
}
catch (System.NotSupportedException e)
{
if (!warned)
{
System.Console.Error.WriteLine("WARNING: " + TestName() + " cannot fully test values of " + q);
warned = true;
}
}
}
}
// verify new values are reloaded (not reused) for a new reader
s = new IndexSearcher(dir);
FieldScoreQuery q2 = new FieldScoreQuery(field, tp);
ScoreDoc[] h2 = s.Search(q2, null, 1000).scoreDocs;
Assert.AreEqual(N_DOCS, h2.Length, "All docs should be matched!");
IndexReader[] readers2 = s.GetIndexReader().GetSequentialSubReaders();
for (int j = 0; j < readers2.Length; j++)
{
IndexReader reader = readers2[j];
try
{
Log("compare: " + innerArray + " to " + q2.valSrc_ForNUnit.GetValues(reader).GetInnerArray());
Assert.AreNotSame(innerArray, q2.valSrc_ForNUnit.GetValues(reader).GetInnerArray(), "cached field values should not be reused if reader as changed!");
}
catch (System.NotSupportedException e)
{
if (!warned)
{
System.Console.Error.WriteLine("WARNING: " + TestName() + " cannot fully test values of " + q2);
warned = true;
}
}
}
}
public void TestNegativePositions()
{
SinkTokenizer tokens = new SinkTokenizer();
Token t = new Token();
t.SetTermText("a");
t.SetPositionIncrement(0);
tokens.Add(t);
t.SetTermText("b");
t.SetPositionIncrement(1);
tokens.Add(t);
t.SetTermText("c");
tokens.Add(t);
MockRAMDirectory dir = new MockRAMDirectory();
IndexWriter w = new IndexWriter(dir, false, new WhitespaceAnalyzer(), true);
Document doc = new Document();
doc.Add(new Field("field", tokens));
w.AddDocument(doc);
w.Close();
IndexSearcher s = new IndexSearcher(dir);
PhraseQuery pq = new PhraseQuery();
pq.Add(new Term("field", "a"));
pq.Add(new Term("field", "b"));
pq.Add(new Term("field", "c"));
Hits hits = s.Search(pq);
Assert.AreEqual(1, hits.Length());
Query q = new SpanTermQuery(new Term("field", "a"));
hits = s.Search(q);
Assert.AreEqual(1, hits.Length());
TermPositions tps = s.GetIndexReader().TermPositions(new Term("field", "a"));
Assert.IsTrue(tps.Next());
Assert.AreEqual(1, tps.Freq());
Assert.AreEqual(-1, tps.NextPosition());
Assert.IsTrue(_TestUtil.CheckIndex(dir));
s.Close();
dir.Close();
}
/* Open pre-lockless index, add docs, do a delete &
* setNorm, and search */
public virtual void ChangeIndexWithAdds(System.String dirName, bool autoCommit)
{
System.String origDirName = dirName;
dirName = FullDir(dirName);
Directory dir = FSDirectory.Open(new System.IO.FileInfo(dirName));
// open writer
IndexWriter writer = new IndexWriter(dir, autoCommit, new WhitespaceAnalyzer(), false);
// add 10 docs
for (int i = 0; i < 10; i++)
{
AddDoc(writer, 35 + i);
}
// make sure writer sees right total -- writer seems not to know about deletes in .del?
int expected;
if (Compare(origDirName, "24") < 0)
{
expected = 45;
}
else
{
expected = 46;
}
Assert.AreEqual(expected, writer.DocCount(), "wrong doc count");
writer.Close();
// make sure searching sees right # hits
IndexSearcher searcher = new IndexSearcher(dir);
ScoreDoc[] hits = searcher.Search(new TermQuery(new Term("content", "aaa")), null, 1000).ScoreDocs;
Document d = searcher.Doc(hits[0].doc);
Assert.AreEqual("21", d.Get("id"), "wrong first document");
TestHits(hits, 44, searcher.GetIndexReader());
searcher.Close();
// make sure we can do delete & setNorm against this
// pre-lockless segment:
IndexReader reader = IndexReader.Open(dir);
Term searchTerm = new Term("id", "6");
int delCount = reader.DeleteDocuments(searchTerm);
Assert.AreEqual(1, delCount, "wrong delete count");
reader.SetNorm(22, "content", (float)2.0);
reader.Close();
// make sure they "took":
searcher = new IndexSearcher(dir);
hits = searcher.Search(new TermQuery(new Term("content", "aaa")), null, 1000).ScoreDocs;
Assert.AreEqual(43, hits.Length, "wrong number of hits");
d = searcher.Doc(hits[0].doc);
Assert.AreEqual("22", d.Get("id"), "wrong first document");
TestHits(hits, 43, searcher.GetIndexReader());
searcher.Close();
// optimize
writer = new IndexWriter(dir, autoCommit, new WhitespaceAnalyzer(), false);
writer.Optimize();
writer.Close();
searcher = new IndexSearcher(dir);
hits = searcher.Search(new TermQuery(new Term("content", "aaa")), null, 1000).ScoreDocs;
Assert.AreEqual(43, hits.Length, "wrong number of hits");
d = searcher.Doc(hits[0].doc);
TestHits(hits, 43, searcher.GetIndexReader());
Assert.AreEqual("22", d.Get("id"), "wrong first document");
searcher.Close();
dir.Close();
}
// Test that values loaded for FieldScoreQuery are cached properly and consumes the proper RAM resources.
private void DoTestCaching(System.String field, FieldScoreQuery.Type tp)
{
// prepare expected array types for comparison
System.Collections.Hashtable expectedArrayTypes = new System.Collections.Hashtable();
expectedArrayTypes[FieldScoreQuery.Type.BYTE] = new sbyte[0];
expectedArrayTypes[FieldScoreQuery.Type.SHORT] = new short[0];
expectedArrayTypes[FieldScoreQuery.Type.INT] = new int[0];
expectedArrayTypes[FieldScoreQuery.Type.FLOAT] = new float[0];
IndexSearcher s = new IndexSearcher(dir);
System.Object[] innerArray = new Object[s.GetIndexReader().GetSequentialSubReaders().Length];
bool warned = false; // print warning once.
for (int i = 0; i < 10; i++)
{
FieldScoreQuery q = new FieldScoreQuery(field, tp);
ScoreDoc[] h = s.Search(q, null, 1000).scoreDocs;
Assert.AreEqual(N_DOCS, h.Length, "All docs should be matched!");
IndexReader[] readers = s.GetIndexReader().GetSequentialSubReaders();
for (int j = 0; j < readers.Length; j++)
{
IndexReader reader = readers[j];
try
{
if (i == 0)
{
innerArray[j] = q.valSrc_ForNUnit.GetValues(reader).GetInnerArray();
Log(i + ". compare: " + innerArray[j].GetType() + " to " + expectedArrayTypes[tp].GetType());
Assert.AreEqual(innerArray[j].GetType(), expectedArrayTypes[tp].GetType(), "field values should be cached in the correct array type!");
}
else
{
Log(i + ". compare: " + innerArray[j] + " to " + q.valSrc_ForNUnit.GetValues(reader).GetInnerArray());
Assert.AreSame(innerArray[j], q.valSrc_ForNUnit.GetValues(reader).GetInnerArray(), "field values should be cached and reused!");
}
}
catch (System.NotSupportedException e)
{
if (!warned)
{
System.Console.Error.WriteLine("WARNING: " + TestName() + " cannot fully test values of " + q);
warned = true;
}
}
}
}
// verify new values are reloaded (not reused) for a new reader
s = new IndexSearcher(dir);
FieldScoreQuery q2 = new FieldScoreQuery(field, tp);
ScoreDoc[] h2 = s.Search(q2, null, 1000).scoreDocs;
Assert.AreEqual(N_DOCS, h2.Length, "All docs should be matched!");
IndexReader[] readers2 = s.GetIndexReader().GetSequentialSubReaders();
for (int j = 0; j < readers2.Length; j++)
{
IndexReader reader = readers2[j];
try
{
Log("compare: " + innerArray + " to " + q2.valSrc_ForNUnit.GetValues(reader).GetInnerArray());
Assert.AreNotSame(innerArray, q2.valSrc_ForNUnit.GetValues(reader).GetInnerArray(), "cached field values should not be reused if reader as changed!");
}
catch (System.NotSupportedException e)
{
if (!warned)
{
System.Console.Error.WriteLine("WARNING: " + TestName() + " cannot fully test values of " + q2);
warned = true;
}
}
}
}
public virtual void TestNegativePositions()
{
TokenStream tokens = new AnonymousClassTokenStream(this);
MockRAMDirectory dir = new MockRAMDirectory();
IndexWriter w = new IndexWriter(dir, new WhitespaceAnalyzer(), true, IndexWriter.MaxFieldLength.UNLIMITED);
Document doc = new Document();
doc.Add(new Field("field", tokens));
w.AddDocument(doc);
w.Commit();
IndexSearcher s = new IndexSearcher(dir);
PhraseQuery pq = new PhraseQuery();
pq.Add(new Term("field", "a"));
pq.Add(new Term("field", "b"));
pq.Add(new Term("field", "c"));
ScoreDoc[] hits = s.Search(pq, null, 1000).scoreDocs;
Assert.AreEqual(1, hits.Length);
Query q = new SpanTermQuery(new Term("field", "a"));
hits = s.Search(q, null, 1000).scoreDocs;
Assert.AreEqual(1, hits.Length);
TermPositions tps = s.GetIndexReader().TermPositions(new Term("field", "a"));
Assert.IsTrue(tps.Next());
Assert.AreEqual(1, tps.Freq());
Assert.AreEqual(0, tps.NextPosition());
w.Close();
Assert.IsTrue(_TestUtil.CheckIndex(dir));
s.Close();
dir.Close();
}
// Test that values loaded for FieldScoreQuery are cached properly and consumes the proper RAM resources.
private void DoTestCaching(System.String field, bool inOrder)
{
IndexSearcher s = new IndexSearcher(dir);
System.Object innerArray = null;
bool warned = false; // print warning once
for (int i = 0; i < 10; i++)
{
ValueSource vs;
if (inOrder)
{
vs = new OrdFieldSource(field);
}
else
{
vs = new ReverseOrdFieldSource(field);
}
ValueSourceQuery q = new ValueSourceQuery(vs);
ScoreDoc[] h = s.Search(q, null, 1000).scoreDocs;
try
{
Assert.AreEqual(N_DOCS, h.Length, "All docs should be matched!");
IndexReader[] readers = s.GetIndexReader().GetSequentialSubReaders();
for (int j = 0; j < readers.Length; j++)
{
IndexReader reader = readers[j];
if (i == 0)
{
innerArray = q.valSrc_ForNUnit.GetValues(reader).GetInnerArray();
}
else
{
Log(i + ". compare: " + innerArray + " to " + q.valSrc_ForNUnit.GetValues(reader).GetInnerArray());
Assert.AreSame(innerArray, q.valSrc_ForNUnit.GetValues(reader).GetInnerArray(), "field values should be cached and reused!");
}
}
}
catch (System.NotSupportedException e)
{
if (!warned)
{
System.Console.Error.WriteLine("WARNING: " + TestName() + " cannot fully test values of " + q);
warned = true;
}
}
}
ValueSource vs2;
ValueSourceQuery q2;
ScoreDoc[] h2;
// verify that different values are loaded for a different field
System.String field2 = INT_FIELD;
Assert.IsFalse(field.Equals(field2)); // otherwise this test is meaningless.
if (inOrder)
{
vs2 = new OrdFieldSource(field2);
}
else
{
vs2 = new ReverseOrdFieldSource(field2);
}
q2 = new ValueSourceQuery(vs2);
h2 = s.Search(q2, null, 1000).scoreDocs;
Assert.AreEqual(N_DOCS, h2.Length, "All docs should be matched!");
IndexReader[] readers2 = s.GetIndexReader().GetSequentialSubReaders();
for (int j = 0; j < readers2.Length; j++)
{
IndexReader reader = readers2[j];
try
{
Log("compare (should differ): " + innerArray + " to " + q2.valSrc_ForNUnit.GetValues(reader).GetInnerArray());
Assert.AreNotSame(innerArray, q2.valSrc_ForNUnit.GetValues(reader).GetInnerArray(), "different values shuold be loaded for a different field!");
}
catch (System.NotSupportedException e)
{
if (!warned)
{
System.Console.Error.WriteLine("WARNING: " + TestName() + " cannot fully test values of " + q2);
warned = true;
}
}
}
// verify new values are reloaded (not reused) for a new reader
s = new IndexSearcher(dir);
if (inOrder)
{
vs2 = new OrdFieldSource(field);
}
else
{
vs2 = new ReverseOrdFieldSource(field);
}
q2 = new ValueSourceQuery(vs2);
h2 = s.Search(q2, null, 1000).scoreDocs;
Assert.AreEqual(N_DOCS, h2.Length, "All docs should be matched!");
readers2 = s.GetIndexReader().GetSequentialSubReaders();
for (int j = 0; j < readers2.Length; j++)
{
IndexReader reader = readers2[j];
try
{
Log("compare (should differ): " + innerArray + " to " + q2.valSrc_ForNUnit.GetValues(reader).GetInnerArray());
Assert.AreNotSame(innerArray, q2.valSrc_ForNUnit.GetValues(reader).GetInnerArray(), "cached field values should not be reused if reader as changed!");
}
catch (System.NotSupportedException e)
{
if (!warned)
{
System.Console.Error.WriteLine("WARNING: " + TestName() + " cannot fully test values of " + q2);
warned = true;
}
}
}
}
public virtual void searchIndex(System.String dirName, System.String oldName)
{
//QueryParser parser = new QueryParser("contents", new WhitespaceAnalyzer());
//Query query = parser.parse("handle:1");
dirName = FullDir(dirName);
Directory dir = FSDirectory.Open(new System.IO.FileInfo(dirName));
IndexSearcher searcher = new IndexSearcher(dir);
IndexReader reader = searcher.GetIndexReader();
_TestUtil.CheckIndex(dir);
for (int i = 0; i < 35; i++)
{
if (!reader.IsDeleted(i))
{
Document d = reader.Document(i);
System.Collections.IList fields = d.GetFields();
if (!oldName.StartsWith("19.") && !oldName.StartsWith("20.") && !oldName.StartsWith("21.") && !oldName.StartsWith("22."))
{
if (d.GetField("content3") == null)
{
Assert.AreEqual(5, fields.Count);
Field f = (Field)d.GetField("id");
Assert.AreEqual("" + i, f.StringValue());
f = (Field)d.GetField("utf8");
Assert.AreEqual("Lu\uD834\uDD1Ece\uD834\uDD60ne \u0000 \u2620 ab\ud917\udc17cd", f.StringValue());
f = (Field)d.GetField("autf8");
Assert.AreEqual("Lu\uD834\uDD1Ece\uD834\uDD60ne \u0000 \u2620 ab\ud917\udc17cd", f.StringValue());
f = (Field)d.GetField("content2");
Assert.AreEqual("here is more content with aaa aaa aaa", f.StringValue());
f = (Field)d.GetField("fie\u2C77ld");
Assert.AreEqual("field with non-ascii name", f.StringValue());
}
}
}
// Only ID 7 is deleted
else
{
Assert.AreEqual(7, i);
}
}
ScoreDoc[] hits = searcher.Search(new TermQuery(new Term("content", "aaa")), null, 1000).ScoreDocs;
// First document should be #21 since it's norm was
// increased:
Document d2 = searcher.Doc(hits[0].doc);
Assert.AreEqual("21", d2.Get("id"), "didn't get the right document first");
TestHits(hits, 34, searcher.GetIndexReader());
if (!oldName.StartsWith("19.") && !oldName.StartsWith("20.") && !oldName.StartsWith("21.") && !oldName.StartsWith("22."))
{
// Test on indices >= 2.3
hits = searcher.Search(new TermQuery(new Term("utf8", "\u0000")), null, 1000).ScoreDocs;
Assert.AreEqual(34, hits.Length);
hits = searcher.Search(new TermQuery(new Term("utf8", "Lu\uD834\uDD1Ece\uD834\uDD60ne")), null, 1000).ScoreDocs;
Assert.AreEqual(34, hits.Length);
hits = searcher.Search(new TermQuery(new Term("utf8", "ab\ud917\udc17cd")), null, 1000).ScoreDocs;
Assert.AreEqual(34, hits.Length);
}
searcher.Close();
dir.Close();
}
// verify results are as expected.
private void VerifyResults(float boost, IndexSearcher s, System.Collections.Hashtable h1, System.Collections.Hashtable h2customNeutral, System.Collections.Hashtable h3CustomMul, System.Collections.Hashtable h4CustomAdd, System.Collections.Hashtable h5CustomMulAdd, Query q1, Query q2, Query q3, Query q4, Query q5)
{
// verify numbers of matches
Log("#hits = " + h1.Count);
Assert.AreEqual(h1.Count, h2customNeutral.Count, "queries should have same #hits");
Assert.AreEqual(h1.Count, h3CustomMul.Count, "queries should have same #hits");
Assert.AreEqual(h1.Count, h4CustomAdd.Count, "queries should have same #hits");
Assert.AreEqual(h1.Count, h5CustomMulAdd.Count, "queries should have same #hits");
// verify scores ratios
for (System.Collections.IEnumerator it = h1.Keys.GetEnumerator(); it.MoveNext(); )
{
System.Int32 x = (System.Int32) it.Current;
int doc = x;
Log("doc = " + doc);
float fieldScore = ExpectedFieldScore(s.GetIndexReader().Document(doc).Get(ID_FIELD));
Log("fieldScore = " + fieldScore);
Assert.IsTrue(fieldScore > 0, "fieldScore should not be 0");
float score1 = (float) ((System.Single) h1[x]);
LogResult("score1=", s, q1, doc, score1);
float score2 = (float) ((System.Single) h2customNeutral[x]);
LogResult("score2=", s, q2, doc, score2);
Assert.AreEqual(boost * score1, score2, TEST_SCORE_TOLERANCE_DELTA, "same score (just boosted) for neutral");
float score3 = (float) ((System.Single) h3CustomMul[x]);
LogResult("score3=", s, q3, doc, score3);
Assert.AreEqual(boost * fieldScore * score1, score3, TEST_SCORE_TOLERANCE_DELTA, "new score for custom mul");
float score4 = (float) ((System.Single) h4CustomAdd[x]);
LogResult("score4=", s, q4, doc, score4);
Assert.AreEqual(boost * (fieldScore + score1), score4, TEST_SCORE_TOLERANCE_DELTA, "new score for custom add");
float score5 = (float) ((System.Single) h5CustomMulAdd[x]);
LogResult("score5=", s, q5, doc, score5);
Assert.AreEqual(boost * fieldScore * (score1 + fieldScore), score5, TEST_SCORE_TOLERANCE_DELTA, "new score for custom mul add");
}
}