public virtual void TestSimple()
{
int numNodes = TestUtil.NextInt32(Random, 1, 10);
double runTimeSec = AtLeast(3);
int minDocsToMakeTerms = TestUtil.NextInt32(Random, 5, 20);
int maxSearcherAgeSeconds = TestUtil.NextInt32(Random, 1, 3);
if (VERBOSE)
{
Console.WriteLine("TEST: numNodes=" + numNodes + " runTimeSec=" + runTimeSec + " maxSearcherAgeSeconds=" + maxSearcherAgeSeconds);
}
Start(numNodes, runTimeSec, maxSearcherAgeSeconds);
List <PreviousSearchState> priorSearches = new List <PreviousSearchState>();
List <BytesRef> terms = null;
while (Time.NanoTime() < endTimeNanos)
{
bool doFollowon = priorSearches.Count > 0 && Random.Next(7) == 1;
// Pick a random node; we will run the query on this node:
int myNodeID = Random.Next(numNodes);
NodeState.ShardIndexSearcher localShardSearcher;
PreviousSearchState prevSearchState;
if (doFollowon)
{
// Pretend user issued a followon query:
prevSearchState = priorSearches[Random.Next(priorSearches.Count)];
if (VERBOSE)
{
Console.WriteLine("\nTEST: follow-on query age=" + ((Time.NanoTime() - prevSearchState.SearchTimeNanos) / 1000000000.0));
}
try
{
localShardSearcher = m_nodes[myNodeID].Acquire(prevSearchState.Versions);
}
catch (SearcherExpiredException see)
{
// Expected, sometimes; in a "real" app we would
// either forward this error to the user ("too
// much time has passed; please re-run your
// search") or sneakily just switch to newest
// searcher w/o telling them...
if (VERBOSE)
{
Console.WriteLine(" searcher expired during local shard searcher init: " + see);
}
priorSearches.Remove(prevSearchState);
continue;
}
}
else
{
if (VERBOSE)
{
Console.WriteLine("\nTEST: fresh query");
}
// Do fresh query:
localShardSearcher = m_nodes[myNodeID].Acquire();
prevSearchState = null;
}
IndexReader[] subs = new IndexReader[numNodes];
PreviousSearchState searchState = null;
try
{
// Mock: now make a single reader (MultiReader) from all node
// searchers. In a real shard env you can't do this... we
// do it to confirm results from the shard searcher
// are correct:
int docCount = 0;
try
{
for (int nodeID = 0; nodeID < numNodes; nodeID++)
{
long subVersion = localShardSearcher.GetNodeVersions()[nodeID];
IndexSearcher sub = m_nodes[nodeID].Searchers.Acquire(subVersion);
if (sub == null)
{
nodeID--;
while (nodeID >= 0)
{
subs[nodeID].DecRef();
subs[nodeID] = null;
nodeID--;
}
throw new SearcherExpiredException("nodeID=" + nodeID + " version=" + subVersion);
}
subs[nodeID] = sub.IndexReader;
docCount += subs[nodeID].MaxDoc;
}
}
catch (SearcherExpiredException see)
{
// Expected
if (VERBOSE)
{
Console.WriteLine(" searcher expired during mock reader init: " + see);
}
continue;
}
IndexReader mockReader = new MultiReader(subs);
IndexSearcher mockSearcher = new IndexSearcher(mockReader);
Query query;
Sort sort;
if (prevSearchState != null)
{
query = prevSearchState.Query;
sort = prevSearchState.Sort;
}
else
{
if (terms == null && docCount > minDocsToMakeTerms)
{
// TODO: try to "focus" on high freq terms sometimes too
// TODO: maybe also periodically reset the terms...?
TermsEnum termsEnum = MultiFields.GetTerms(mockReader, "body").GetIterator(null);
terms = new List <BytesRef>();
while (termsEnum.Next() != null)
{
terms.Add(BytesRef.DeepCopyOf(termsEnum.Term));
}
if (VERBOSE)
{
Console.WriteLine("TEST: init terms: " + terms.Count + " terms");
}
if (terms.Count == 0)
{
terms = null;
}
}
if (VERBOSE)
{
Console.WriteLine(" maxDoc=" + mockReader.MaxDoc);
}
if (terms != null)
{
if (Random.NextBoolean())
{
query = new TermQuery(new Term("body", terms[Random.Next(terms.Count)]));
}
else
{
string t = terms[Random.Next(terms.Count)].Utf8ToString();
string prefix;
if (t.Length <= 1)
{
prefix = t;
}
else
{
prefix = t.Substring(0, TestUtil.NextInt32(Random, 1, 2));
}
query = new PrefixQuery(new Term("body", prefix));
}
if (Random.NextBoolean())
{
sort = null;
}
else
{
// TODO: sort by more than 1 field
int what = Random.Next(3);
if (what == 0)
{
sort = new Sort(SortField.FIELD_SCORE);
}
else if (what == 1)
{
// TODO: this sort doesn't merge
// correctly... it's tricky because you
// could have > 2.1B docs across all shards:
//sort = new Sort(SortField.FIELD_DOC);
sort = null;
}
else if (what == 2)
{
sort = new Sort(new SortField[] { new SortField("docid", SortFieldType.INT32, Random.NextBoolean()) });
}
else
{
sort = new Sort(new SortField[] { new SortField("title", SortFieldType.STRING, Random.NextBoolean()) });
}
}
}
else
{
query = null;
sort = null;
}
}
if (query != null)
{
try
{
searchState = AssertSame(mockSearcher, localShardSearcher, query, sort, prevSearchState);
}
catch (SearcherExpiredException see)
{
// Expected; in a "real" app we would
// either forward this error to the user ("too
// much time has passed; please re-run your
// search") or sneakily just switch to newest
// searcher w/o telling them...
if (VERBOSE)
{
Console.WriteLine(" searcher expired during search: " + see);
Console.Out.Write(see.StackTrace);
}
// We can't do this in general: on a very slow
// computer it's possible the local searcher
// expires before we can finish our search:
// assert prevSearchState != null;
if (prevSearchState != null)
{
priorSearches.Remove(prevSearchState);
}
}
}
}
finally
{
m_nodes[myNodeID].Release(localShardSearcher);
foreach (IndexReader sub in subs)
{
if (sub != null)
{
sub.DecRef();
}
}
}
if (searchState != null && searchState.SearchAfterLocal != null && Random.Next(5) == 3)
{
priorSearches.Add(searchState);
if (priorSearches.Count > 200)
{
priorSearches.Shuffle(Random);
priorSearches.SubList(100, priorSearches.Count).Clear();
}
}
}
Finish();
}
private void TestRandomTrieAndClassicRangeQuery(int precisionStep)
{
string field = "field" + precisionStep;
int totalTermCountT = 0, totalTermCountC = 0, termCountT, termCountC;
int num = TestUtil.NextInt32(Random, 10, 20);
for (int i = 0; i < num; i++)
{
long lower = (long)(Random.NextDouble() * NoDocs * Distance) + StartOffset;
long upper = (long)(Random.NextDouble() * NoDocs * Distance) + StartOffset;
if (lower > upper)
{
long a = lower;
lower = upper;
upper = a;
}
BytesRef lowerBytes = new BytesRef(NumericUtils.BUF_SIZE_INT64), upperBytes = new BytesRef(NumericUtils.BUF_SIZE_INT64);
NumericUtils.Int64ToPrefixCodedBytes(lower, 0, lowerBytes);
NumericUtils.Int64ToPrefixCodedBytes(upper, 0, upperBytes);
// test inclusive range
NumericRangeQuery <long> tq = NumericRangeQuery.NewInt64Range(field, precisionStep, lower, upper, true, true);
TermRangeQuery cq = new TermRangeQuery(field, lowerBytes, upperBytes, true, true);
TopDocs tTopDocs = Searcher.Search(tq, 1);
TopDocs cTopDocs = Searcher.Search(cq, 1);
Assert.AreEqual(cTopDocs.TotalHits, tTopDocs.TotalHits, "Returned count for NumericRangeQuery and TermRangeQuery must be equal");
totalTermCountT += termCountT = CountTerms(tq);
totalTermCountC += termCountC = CountTerms(cq);
CheckTermCounts(precisionStep, termCountT, termCountC);
// test exclusive range
tq = NumericRangeQuery.NewInt64Range(field, precisionStep, lower, upper, false, false);
cq = new TermRangeQuery(field, lowerBytes, upperBytes, false, false);
tTopDocs = Searcher.Search(tq, 1);
cTopDocs = Searcher.Search(cq, 1);
Assert.AreEqual(cTopDocs.TotalHits, tTopDocs.TotalHits, "Returned count for NumericRangeQuery and TermRangeQuery must be equal");
totalTermCountT += termCountT = CountTerms(tq);
totalTermCountC += termCountC = CountTerms(cq);
CheckTermCounts(precisionStep, termCountT, termCountC);
// test left exclusive range
tq = NumericRangeQuery.NewInt64Range(field, precisionStep, lower, upper, false, true);
cq = new TermRangeQuery(field, lowerBytes, upperBytes, false, true);
tTopDocs = Searcher.Search(tq, 1);
cTopDocs = Searcher.Search(cq, 1);
Assert.AreEqual(cTopDocs.TotalHits, tTopDocs.TotalHits, "Returned count for NumericRangeQuery and TermRangeQuery must be equal");
totalTermCountT += termCountT = CountTerms(tq);
totalTermCountC += termCountC = CountTerms(cq);
CheckTermCounts(precisionStep, termCountT, termCountC);
// test right exclusive range
tq = NumericRangeQuery.NewInt64Range(field, precisionStep, lower, upper, true, false);
cq = new TermRangeQuery(field, lowerBytes, upperBytes, true, false);
tTopDocs = Searcher.Search(tq, 1);
cTopDocs = Searcher.Search(cq, 1);
Assert.AreEqual(cTopDocs.TotalHits, tTopDocs.TotalHits, "Returned count for NumericRangeQuery and TermRangeQuery must be equal");
totalTermCountT += termCountT = CountTerms(tq);
totalTermCountC += termCountC = CountTerms(cq);
CheckTermCounts(precisionStep, termCountT, termCountC);
}
CheckTermCounts(precisionStep, totalTermCountT, totalTermCountC);
if (VERBOSE && precisionStep != int.MaxValue)
{
Console.WriteLine("Average number of terms during random search on '" + field + "':");
Console.WriteLine(" Numeric query: " + (((double)totalTermCountT) / (num * 4)));
Console.WriteLine(" Classical query: " + (((double)totalTermCountC) / (num * 4)));
}
}
public virtual void TestRandom()
{
AssumeTrue("Test requires SortedSetDV support", DefaultCodecSupportsSortedSet);
string[] tokens = GetRandomTokens(10);
Directory indexDir = NewDirectory();
Directory taxoDir = NewDirectory();
RandomIndexWriter w = new RandomIndexWriter(
#if FEATURE_INSTANCE_TESTDATA_INITIALIZATION
this,
#endif
Random, indexDir);
FacetsConfig config = new FacetsConfig();
int numDocs = AtLeast(1000);
int numDims = TestUtil.NextInt32(Random, 1, 7);
IList <TestDoc> testDocs = GetRandomDocs(tokens, numDocs, numDims);
foreach (TestDoc testDoc in testDocs)
{
Document doc = new Document();
doc.Add(NewStringField("content", testDoc.content, Field.Store.NO));
for (int j = 0; j < numDims; j++)
{
if (testDoc.dims[j] != null)
{
doc.Add(new SortedSetDocValuesFacetField("dim" + j, testDoc.dims[j]));
}
}
w.AddDocument(config.Build(doc));
}
// NRT open
IndexSearcher searcher = NewSearcher(w.GetReader());
// Per-top-reader state:
SortedSetDocValuesReaderState state = new DefaultSortedSetDocValuesReaderState(searcher.IndexReader);
int iters = AtLeast(100);
for (int iter = 0; iter < iters; iter++)
{
string searchToken = tokens[Random.Next(tokens.Length)];
if (VERBOSE)
{
Console.WriteLine("\nTEST: iter content=" + searchToken);
}
FacetsCollector fc = new FacetsCollector();
FacetsCollector.Search(searcher, new TermQuery(new Term("content", searchToken)), 10, fc);
Facets facets = new SortedSetDocValuesFacetCounts(state, fc);
// Slow, yet hopefully bug-free, faceting:
var expectedCounts = new List <Dictionary <string, int?> >();
for (int i = 0; i < numDims; i++)
{
expectedCounts.Add(new Dictionary <string, int?>());
}
foreach (TestDoc doc in testDocs)
{
if (doc.content.Equals(searchToken, StringComparison.Ordinal))
{
for (int j = 0; j < numDims; j++)
{
if (doc.dims[j] != null)
{
int?v;
if (!expectedCounts[j].TryGetValue(doc.dims[j], out v))
{
expectedCounts[j][doc.dims[j]] = 1;
}
else
{
expectedCounts[j][doc.dims[j]] = (int)v + 1;
}
}
}
}
}
List <FacetResult> expected = new List <FacetResult>();
for (int i = 0; i < numDims; i++)
{
List <LabelAndValue> labelValues = new List <LabelAndValue>();
int totCount = 0;
foreach (KeyValuePair <string, int?> ent in expectedCounts[i])
{
labelValues.Add(new LabelAndValue(ent.Key, ent.Value.Value));
totCount += ent.Value.Value;
}
SortLabelValues(labelValues);
if (totCount > 0)
{
expected.Add(new FacetResult("dim" + i, new string[0], totCount, labelValues.ToArray(), labelValues.Count));
}
}
// Sort by highest value, tie break by value:
SortFacetResults(expected);
IList <FacetResult> actual = facets.GetAllDims(10);
// Messy: fixup ties
//sortTies(actual);
CollectionAssert.AreEqual(expected, actual);
}
IOUtils.Dispose(w, searcher.IndexReader, indexDir, taxoDir);
}
public virtual void TestRandom()
{
int iters = AtLeast(10);
for (int iter = 0; iter < iters; iter++)
{
int numBytes = TestUtil.NextInt32(Random, 1, 200000);
byte[] expected = new byte[numBytes];
int blockBits = TestUtil.NextInt32(Random, 8, 15);
BytesStore bytes = new BytesStore(blockBits);
if (VERBOSE)
{
Console.WriteLine("TEST: iter=" + iter + " numBytes=" + numBytes + " blockBits=" + blockBits);
}
int pos = 0;
while (pos < numBytes)
{
int op = Random.Next(8);
if (VERBOSE)
{
Console.WriteLine(" cycle pos=" + pos);
}
switch (op)
{
case 0:
{
// write random byte
byte b = (byte)Random.Next(256);
if (VERBOSE)
{
Console.WriteLine(" writeByte b=" + b);
}
expected[pos++] = b;
bytes.WriteByte(b);
}
break;
case 1:
{
// write random byte[]
int len = Random.Next(Math.Min(numBytes - pos, 100));
byte[] temp = new byte[len];
Random.NextBytes(temp);
if (VERBOSE)
{
Console.WriteLine(" writeBytes len=" + len + " bytes=" + Arrays.ToString(temp));
}
Array.Copy(temp, 0, expected, pos, temp.Length);
bytes.WriteBytes(temp, 0, temp.Length);
pos += len;
}
break;
case 2:
{
// write int @ absolute pos
if (pos > 4)
{
int x = Random.Next();
int randomPos = Random.Next(pos - 4);
if (VERBOSE)
{
Console.WriteLine(" abs writeInt pos=" + randomPos + " x=" + x);
}
bytes.WriteInt32(randomPos, x);
expected[randomPos++] = (byte)(x >> 24);
expected[randomPos++] = (byte)(x >> 16);
expected[randomPos++] = (byte)(x >> 8);
expected[randomPos++] = (byte)x;
}
}
break;
case 3:
{
// reverse bytes
if (pos > 1)
{
int len = TestUtil.NextInt32(Random, 2, Math.Min(100, pos));
int start;
if (len == pos)
{
start = 0;
}
else
{
start = Random.Next(pos - len);
}
int end = start + len - 1;
if (VERBOSE)
{
Console.WriteLine(" reverse start=" + start + " end=" + end + " len=" + len + " pos=" + pos);
}
bytes.Reverse(start, end);
while (start <= end)
{
byte b = expected[end];
expected[end] = expected[start];
expected[start] = b;
start++;
end--;
}
}
}
break;
case 4:
{
// abs write random byte[]
if (pos > 2)
{
int randomPos = Random.Next(pos - 1);
int len = TestUtil.NextInt32(Random, 1, Math.Min(pos - randomPos - 1, 100));
byte[] temp = new byte[len];
Random.NextBytes(temp);
if (VERBOSE)
{
Console.WriteLine(" abs writeBytes pos=" + randomPos + " len=" + len + " bytes=" + Arrays.ToString(temp));
}
Array.Copy(temp, 0, expected, randomPos, temp.Length);
bytes.WriteBytes(randomPos, temp, 0, temp.Length);
}
}
break;
case 5:
{
// copyBytes
if (pos > 1)
{
int src = Random.Next(pos - 1);
int dest = TestUtil.NextInt32(Random, src + 1, pos - 1);
int len = TestUtil.NextInt32(Random, 1, Math.Min(300, pos - dest));
if (VERBOSE)
{
Console.WriteLine(" copyBytes src=" + src + " dest=" + dest + " len=" + len);
}
Array.Copy(expected, src, expected, dest, len);
bytes.CopyBytes(src, dest, len);
}
}
break;
case 6:
{
// skip
int len = Random.Next(Math.Min(100, numBytes - pos));
if (VERBOSE)
{
Console.WriteLine(" skip len=" + len);
}
pos += len;
bytes.SkipBytes(len);
// NOTE: must fill in zeros in case truncate was
// used, else we get false fails:
if (len > 0)
{
byte[] zeros = new byte[len];
bytes.WriteBytes(pos - len, zeros, 0, len);
}
}
break;
case 7:
{
// absWriteByte
if (pos > 0)
{
int dest = Random.Next(pos);
byte b = (byte)Random.Next(256);
expected[dest] = b;
bytes.WriteByte(dest, b);
}
break;
}
}
Assert.AreEqual(pos, bytes.Position);
if (pos > 0 && Random.Next(50) == 17)
{
// truncate
int len = TestUtil.NextInt32(Random, 1, Math.Min(pos, 100));
bytes.Truncate(pos - len);
pos -= len;
Arrays.Fill(expected, pos, pos + len, (byte)0);
if (VERBOSE)
{
Console.WriteLine(" truncate len=" + len + " newPos=" + pos);
}
}
if ((pos > 0 && Random.Next(200) == 17))
{
Verify(bytes, expected, pos);
}
}
BytesStore bytesToVerify;
if (Random.NextBoolean())
{
if (VERBOSE)
{
Console.WriteLine("TEST: save/load final bytes");
}
Directory dir = NewDirectory();
IndexOutput @out = dir.CreateOutput("bytes", IOContext.DEFAULT);
bytes.WriteTo(@out);
@out.Dispose();
IndexInput @in = dir.OpenInput("bytes", IOContext.DEFAULT);
bytesToVerify = new BytesStore(@in, numBytes, TestUtil.NextInt32(Random, 256, int.MaxValue));
@in.Dispose();
dir.Dispose();
}
else
{
bytesToVerify = bytes;
}
Verify(bytesToVerify, expected, numBytes);
}
}
public virtual void TestArbitraryFields()
{
Directory dir = NewDirectory();
RandomIndexWriter w = new RandomIndexWriter(
#if FEATURE_INSTANCE_TESTDATA_INITIALIZATION
this,
#endif
Random, dir);
int NUM_DOCS = AtLeast(27);
if (Verbose)
{
Console.WriteLine("TEST: " + NUM_DOCS + " docs");
}
int[] fieldsPerDoc = new int[NUM_DOCS];
int baseCount = 0;
for (int docCount = 0; docCount < NUM_DOCS; docCount++)
{
int fieldCount = TestUtil.NextInt32(Random, 1, 17);
fieldsPerDoc[docCount] = fieldCount - 1;
int finalDocCount = docCount;
if (Verbose)
{
Console.WriteLine("TEST: " + fieldCount + " fields in doc " + docCount);
}
int finalBaseCount = baseCount;
baseCount += fieldCount - 1;
w.AddDocument(new IterableAnonymousInnerClassHelper(this, fieldCount, finalDocCount, finalBaseCount));
}
IndexReader r = w.GetReader();
w.Dispose();
IndexSearcher s = NewSearcher(r);
int counter = 0;
for (int id = 0; id < NUM_DOCS; id++)
{
if (Verbose)
{
Console.WriteLine("TEST: verify doc id=" + id + " (" + fieldsPerDoc[id] + " fields) counter=" + counter);
}
TopDocs hits = s.Search(new TermQuery(new Term("id", "" + id)), 1);
Assert.AreEqual(1, hits.TotalHits);
int docID = hits.ScoreDocs[0].Doc;
Document doc = s.Doc(docID);
int endCounter = counter + fieldsPerDoc[id];
while (counter < endCounter)
{
string name = "f" + counter;
int fieldID = counter % 10;
bool stored = (counter & 1) == 0 || fieldID == 3;
bool binary = fieldID == 3;
bool indexed = fieldID != 3;
string stringValue;
if (fieldID != 3 && fieldID != 9)
{
stringValue = "text " + counter;
}
else
{
stringValue = null;
}
// stored:
if (stored)
{
IIndexableField f = doc.GetField(name);
Assert.IsNotNull(f, "doc " + id + " doesn't have field f" + counter);
if (binary)
{
Assert.IsNotNull(f, "doc " + id + " doesn't have field f" + counter);
BytesRef b = f.GetBinaryValue();
Assert.IsNotNull(b);
Assert.AreEqual(10, b.Length);
for (int idx = 0; idx < 10; idx++)
{
Assert.AreEqual((byte)(idx + counter), b.Bytes[b.Offset + idx]);
}
}
else
{
if (Debugging.AssertsEnabled)
{
Debugging.Assert(stringValue != null);
}
Assert.AreEqual(stringValue, f.GetStringValue());
}
}
if (indexed)
{
bool tv = counter % 2 == 1 && fieldID != 9;
if (tv)
{
Terms tfv = r.GetTermVectors(docID).GetTerms(name);
Assert.IsNotNull(tfv);
TermsEnum termsEnum = tfv.GetIterator(null);
Assert.AreEqual(new BytesRef("" + counter), termsEnum.Next());
Assert.AreEqual(1, termsEnum.TotalTermFreq);
DocsAndPositionsEnum dpEnum = termsEnum.DocsAndPositions(null, null);
Assert.IsTrue(dpEnum.NextDoc() != DocIdSetIterator.NO_MORE_DOCS);
Assert.AreEqual(1, dpEnum.Freq);
Assert.AreEqual(1, dpEnum.NextPosition());
Assert.AreEqual(new BytesRef("text"), termsEnum.Next());
Assert.AreEqual(1, termsEnum.TotalTermFreq);
dpEnum = termsEnum.DocsAndPositions(null, dpEnum);
Assert.IsTrue(dpEnum.NextDoc() != DocIdSetIterator.NO_MORE_DOCS);
Assert.AreEqual(1, dpEnum.Freq);
Assert.AreEqual(0, dpEnum.NextPosition());
Assert.IsNull(termsEnum.Next());
// TODO: offsets
}
else
{
Fields vectors = r.GetTermVectors(docID);
Assert.IsTrue(vectors == null || vectors.GetTerms(name) == null);
}
BooleanQuery bq = new BooleanQuery();
bq.Add(new TermQuery(new Term("id", "" + id)), Occur.MUST);
bq.Add(new TermQuery(new Term(name, "text")), Occur.MUST);
TopDocs hits2 = s.Search(bq, 1);
Assert.AreEqual(1, hits2.TotalHits);
Assert.AreEqual(docID, hits2.ScoreDocs[0].Doc);
bq = new BooleanQuery();
bq.Add(new TermQuery(new Term("id", "" + id)), Occur.MUST);
bq.Add(new TermQuery(new Term(name, "" + counter)), Occur.MUST);
TopDocs hits3 = s.Search(bq, 1);
Assert.AreEqual(1, hits3.TotalHits);
Assert.AreEqual(docID, hits3.ScoreDocs[0].Doc);
}
counter++;
}
}
r.Dispose();
dir.Dispose();
}
public virtual void TestIntersectRandom()
{
Directory dir = NewDirectory();
RandomIndexWriter w = new RandomIndexWriter(
#if FEATURE_INSTANCE_TESTDATA_INITIALIZATION
this,
#endif
Random, dir);
int numTerms = AtLeast(300);
//final int numTerms = 50;
ISet <string> terms = new JCG.HashSet <string>();
ICollection <string> pendingTerms = new List <string>();
IDictionary <BytesRef, int?> termToID = new Dictionary <BytesRef, int?>();
int id = 0;
while (terms.Count != numTerms)
{
string s = RandomString;
if (!terms.Contains(s))
{
terms.Add(s);
pendingTerms.Add(s);
if (Random.Next(20) == 7)
{
AddDoc(w, pendingTerms, termToID, id++);
}
}
}
AddDoc(w, pendingTerms, termToID, id++);
BytesRef[] termsArray = new BytesRef[terms.Count];
ISet <BytesRef> termsSet = new JCG.HashSet <BytesRef>();
{
int upto = 0;
foreach (string s in terms)
{
BytesRef b = new BytesRef(s);
termsArray[upto++] = b;
termsSet.Add(b);
}
Array.Sort(termsArray);
}
if (VERBOSE)
{
Console.WriteLine("\nTEST: indexed terms (unicode order):");
foreach (BytesRef t in termsArray)
{
Console.WriteLine(" " + t.Utf8ToString() + " -> id:" + termToID[t]);
}
}
IndexReader r = w.GetReader();
w.Dispose();
// NOTE: intentional insanity!!
FieldCache.Int32s docIDToID = FieldCache.DEFAULT.GetInt32s(SlowCompositeReaderWrapper.Wrap(r), "id", false);
for (int iter = 0; iter < 10 * RANDOM_MULTIPLIER; iter++)
{
// TODO: can we also test infinite As here...?
// From the random terms, pick some ratio and compile an
// automaton:
ISet <string> acceptTerms = new JCG.HashSet <string>();
JCG.SortedSet <BytesRef> sortedAcceptTerms = new JCG.SortedSet <BytesRef>();
double keepPct = Random.NextDouble();
Automaton a;
if (iter == 0)
{
if (VERBOSE)
{
Console.WriteLine("\nTEST: empty automaton");
}
a = BasicAutomata.MakeEmpty();
}
else
{
if (VERBOSE)
{
Console.WriteLine("\nTEST: keepPct=" + keepPct);
}
foreach (string s in terms)
{
string s2;
if (Random.NextDouble() <= keepPct)
{
s2 = s;
}
else
{
s2 = RandomString;
}
acceptTerms.Add(s2);
sortedAcceptTerms.Add(new BytesRef(s2));
}
a = BasicAutomata.MakeStringUnion(sortedAcceptTerms);
}
if (Random.NextBoolean())
{
if (VERBOSE)
{
Console.WriteLine("TEST: reduce the automaton");
}
a.Reduce();
}
CompiledAutomaton c = new CompiledAutomaton(a, true, false);
BytesRef[] acceptTermsArray = new BytesRef[acceptTerms.Count];
ISet <BytesRef> acceptTermsSet = new JCG.HashSet <BytesRef>();
int upto = 0;
foreach (string s in acceptTerms)
{
BytesRef b = new BytesRef(s);
acceptTermsArray[upto++] = b;
acceptTermsSet.Add(b);
Assert.IsTrue(Accepts(c, b));
}
Array.Sort(acceptTermsArray);
if (VERBOSE)
{
Console.WriteLine("\nTEST: accept terms (unicode order):");
foreach (BytesRef t in acceptTermsArray)
{
Console.WriteLine(" " + t.Utf8ToString() + (termsSet.Contains(t) ? " (exists)" : ""));
}
Console.WriteLine(a.ToDot());
}
for (int iter2 = 0; iter2 < 100; iter2++)
{
BytesRef startTerm = acceptTermsArray.Length == 0 || Random.NextBoolean() ? null : acceptTermsArray[Random.Next(acceptTermsArray.Length)];
if (VERBOSE)
{
Console.WriteLine("\nTEST: iter2=" + iter2 + " startTerm=" + (startTerm == null ? "<null>" : startTerm.Utf8ToString()));
if (startTerm != null)
{
int state = c.RunAutomaton.InitialState;
for (int idx = 0; idx < startTerm.Length; idx++)
{
int label = startTerm.Bytes[startTerm.Offset + idx] & 0xff;
Console.WriteLine(" state=" + state + " label=" + label);
state = c.RunAutomaton.Step(state, label);
Assert.IsTrue(state != -1);
}
Console.WriteLine(" state=" + state);
}
}
TermsEnum te = MultiFields.GetTerms(r, "f").Intersect(c, startTerm);
int loc;
if (startTerm == null)
{
loc = 0;
}
else
{
loc = Array.BinarySearch(termsArray, BytesRef.DeepCopyOf(startTerm));
if (loc < 0)
{
loc = -(loc + 1);
}
else
{
// startTerm exists in index
loc++;
}
}
while (loc < termsArray.Length && !acceptTermsSet.Contains(termsArray[loc]))
{
loc++;
}
DocsEnum docsEnum = null;
while (loc < termsArray.Length)
{
BytesRef expected = termsArray[loc];
BytesRef actual = te.Next();
if (VERBOSE)
{
Console.WriteLine("TEST: next() expected=" + expected.Utf8ToString() + " actual=" + (actual == null ? "null" : actual.Utf8ToString()));
}
Assert.AreEqual(expected, actual);
Assert.AreEqual(1, te.DocFreq);
docsEnum = TestUtil.Docs(Random, te, null, docsEnum, DocsFlags.NONE);
int docID = docsEnum.NextDoc();
Assert.IsTrue(docID != DocIdSetIterator.NO_MORE_DOCS);
Assert.AreEqual(docIDToID.Get(docID), (int)termToID[expected]);
do
{
loc++;
} while (loc < termsArray.Length && !acceptTermsSet.Contains(termsArray[loc]));
}
Assert.IsNull(te.Next());
}
}
r.Dispose();
dir.Dispose();
}
private void TestRandomSeeks(IndexReader r, params string[] validTermStrings)
{
BytesRef[] validTerms = new BytesRef[validTermStrings.Length];
for (int termIDX = 0; termIDX < validTermStrings.Length; termIDX++)
{
validTerms[termIDX] = new BytesRef(validTermStrings[termIDX]);
}
Array.Sort(validTerms);
if (VERBOSE)
{
Console.WriteLine("TEST: " + validTerms.Length + " terms:");
foreach (BytesRef t in validTerms)
{
Console.WriteLine(" " + t.Utf8ToString() + " " + t);
}
}
TermsEnum te = MultiFields.GetTerms(r, FIELD).GetIterator(null);
int END_LOC = -validTerms.Length - 1;
IList <TermAndState> termStates = new List <TermAndState>();
for (int iter = 0; iter < 100 * RANDOM_MULTIPLIER; iter++)
{
BytesRef t;
int loc;
TermState termState;
if (Random.Next(6) == 4)
{
// pick term that doens't exist:
t = GetNonExistTerm(validTerms);
termState = null;
if (VERBOSE)
{
Console.WriteLine("\nTEST: invalid term=" + t.Utf8ToString());
}
loc = Array.BinarySearch(validTerms, t);
}
else if (termStates.Count != 0 && Random.Next(4) == 1)
{
TermAndState ts = termStates[Random.Next(termStates.Count)];
t = ts.Term;
loc = Array.BinarySearch(validTerms, t);
Assert.IsTrue(loc >= 0);
termState = ts.State;
if (VERBOSE)
{
Console.WriteLine("\nTEST: valid termState term=" + t.Utf8ToString());
}
}
else
{
// pick valid term
loc = Random.Next(validTerms.Length);
t = BytesRef.DeepCopyOf(validTerms[loc]);
termState = null;
if (VERBOSE)
{
Console.WriteLine("\nTEST: valid term=" + t.Utf8ToString());
}
}
// seekCeil or seekExact:
bool doSeekExact = Random.NextBoolean();
if (termState != null)
{
if (VERBOSE)
{
Console.WriteLine(" seekExact termState");
}
te.SeekExact(t, termState);
}
else if (doSeekExact)
{
if (VERBOSE)
{
Console.WriteLine(" seekExact");
}
Assert.AreEqual(loc >= 0, te.SeekExact(t));
}
else
{
if (VERBOSE)
{
Console.WriteLine(" seekCeil");
}
TermsEnum.SeekStatus result = te.SeekCeil(t);
if (VERBOSE)
{
Console.WriteLine(" got " + result);
}
if (loc >= 0)
{
Assert.AreEqual(TermsEnum.SeekStatus.FOUND, result);
}
else if (loc == END_LOC)
{
Assert.AreEqual(TermsEnum.SeekStatus.END, result);
}
else
{
Debug.Assert(loc >= -validTerms.Length);
Assert.AreEqual(TermsEnum.SeekStatus.NOT_FOUND, result);
}
}
if (loc >= 0)
{
Assert.AreEqual(t, te.Term);
}
else if (doSeekExact)
{
// TermsEnum is unpositioned if seekExact returns false
continue;
}
else if (loc == END_LOC)
{
continue;
}
else
{
loc = -loc - 1;
Assert.AreEqual(validTerms[loc], te.Term);
}
// Do a bunch of next's after the seek
int numNext = Random.Next(validTerms.Length);
for (int nextCount = 0; nextCount < numNext; nextCount++)
{
if (VERBOSE)
{
Console.WriteLine("\nTEST: next loc=" + loc + " of " + validTerms.Length);
}
BytesRef t2 = te.Next();
loc++;
if (loc == validTerms.Length)
{
Assert.IsNull(t2);
break;
}
else
{
Assert.AreEqual(validTerms[loc], t2);
if (Random.Next(40) == 17 && termStates.Count < 100)
{
termStates.Add(new TermAndState(validTerms[loc], te.GetTermState()));
}
}
}
}
}
public virtual void TestCustomDoublesValueSource()
{
Directory dir = NewDirectory();
RandomIndexWriter writer = new RandomIndexWriter(
#if FEATURE_INSTANCE_TESTDATA_INITIALIZATION
this,
#endif
Random, dir);
Document doc = new Document();
writer.AddDocument(doc);
writer.AddDocument(doc);
writer.AddDocument(doc);
// Test wants 3 docs in one segment:
writer.ForceMerge(1);
var vs = new ValueSourceAnonymousInnerClassHelper(this, doc);
FacetsConfig config = new FacetsConfig();
FacetsCollector fc = new FacetsCollector();
IndexReader r = writer.GetReader();
IndexSearcher s = NewSearcher(r);
s.Search(new MatchAllDocsQuery(), fc);
DoubleRange[] ranges = new DoubleRange[] { new DoubleRange("< 1", 0.0, true, 1.0, false), new DoubleRange("< 2", 0.0, true, 2.0, false), new DoubleRange("< 5", 0.0, true, 5.0, false), new DoubleRange("< 10", 0.0, true, 10.0, false), new DoubleRange("< 20", 0.0, true, 20.0, false), new DoubleRange("< 50", 0.0, true, 50.0, false) };
Filter fastMatchFilter;
AtomicBoolean filterWasUsed = new AtomicBoolean();
if (Random.NextBoolean())
{
// Sort of silly:
fastMatchFilter = new CachingWrapperFilterAnonymousInnerClassHelper(this, new QueryWrapperFilter(new MatchAllDocsQuery()), filterWasUsed);
}
else
{
fastMatchFilter = null;
}
if (VERBOSE)
{
Console.WriteLine("TEST: fastMatchFilter=" + fastMatchFilter);
}
Facets facets = new DoubleRangeFacetCounts("field", vs, fc, fastMatchFilter, ranges);
Assert.AreEqual("dim=field path=[] value=3 childCount=6\n < 1 (0)\n < 2 (1)\n < 5 (3)\n < 10 (3)\n < 20 (3)\n < 50 (3)\n", facets.GetTopChildren(10, "field").ToString());
Assert.True(fastMatchFilter == null || filterWasUsed);
DrillDownQuery ddq = new DrillDownQuery(config);
ddq.Add("field", ranges[1].GetFilter(fastMatchFilter, vs));
// Test simple drill-down:
Assert.AreEqual(1, s.Search(ddq, 10).TotalHits);
// Test drill-sideways after drill-down
DrillSideways ds = new DrillSidewaysAnonymousInnerClassHelper2(this, s, config, (TaxonomyReader)null, vs, ranges, fastMatchFilter);
DrillSidewaysResult dsr = ds.Search(ddq, 10);
Assert.AreEqual(1, dsr.Hits.TotalHits);
Assert.AreEqual("dim=field path=[] value=3 childCount=6\n < 1 (0)\n < 2 (1)\n < 5 (3)\n < 10 (3)\n < 20 (3)\n < 50 (3)\n", dsr.Facets.GetTopChildren(10, "field").ToString());
IOUtils.Dispose(r, writer, dir);
}
public virtual void TestMixedRangeAndNonRangeTaxonomy()
{
Directory d = NewDirectory();
RandomIndexWriter w = new RandomIndexWriter(
#if FEATURE_INSTANCE_TESTDATA_INITIALIZATION
this,
#endif
Random, d);
Directory td = NewDirectory();
DirectoryTaxonomyWriter tw = new DirectoryTaxonomyWriter(td, OpenMode.CREATE);
FacetsConfig config = new FacetsConfig();
for (long l = 0; l < 100; l++)
{
Document doc = new Document();
// For computing range facet counts:
doc.Add(new NumericDocValuesField("field", l));
// For drill down by numeric range:
doc.Add(new Int64Field("field", l, Field.Store.NO));
if ((l & 3) == 0)
{
doc.Add(new FacetField("dim", "a"));
}
else
{
doc.Add(new FacetField("dim", "b"));
}
w.AddDocument(config.Build(tw, doc));
}
IndexReader r = w.GetReader();
var tr = new DirectoryTaxonomyReader(tw);
IndexSearcher s = NewSearcher(r);
if (VERBOSE)
{
Console.WriteLine("TEST: searcher=" + s);
}
DrillSideways ds = new DrillSidewaysAnonymousInnerClassHelper(this, s, config, tr);
// First search, no drill downs:
DrillDownQuery ddq = new DrillDownQuery(config);
DrillSidewaysResult dsr = ds.Search(null, ddq, 10);
Assert.AreEqual(100, dsr.Hits.TotalHits);
Assert.AreEqual("dim=dim path=[] value=100 childCount=2\n b (75)\n a (25)\n", dsr.Facets.GetTopChildren(10, "dim").ToString());
Assert.AreEqual("dim=field path=[] value=21 childCount=5\n less than 10 (10)\n less than or equal to 10 (11)\n over 90 (9)\n 90 or above (10)\n over 1000 (0)\n", dsr.Facets.GetTopChildren(10, "field").ToString());
// Second search, drill down on dim=b:
ddq = new DrillDownQuery(config);
ddq.Add("dim", "b");
dsr = ds.Search(null, ddq, 10);
Assert.AreEqual(75, dsr.Hits.TotalHits);
Assert.AreEqual("dim=dim path=[] value=100 childCount=2\n b (75)\n a (25)\n", dsr.Facets.GetTopChildren(10, "dim").ToString());
Assert.AreEqual("dim=field path=[] value=16 childCount=5\n less than 10 (7)\n less than or equal to 10 (8)\n over 90 (7)\n 90 or above (8)\n over 1000 (0)\n", dsr.Facets.GetTopChildren(10, "field").ToString());
// Third search, drill down on "less than or equal to 10":
ddq = new DrillDownQuery(config);
ddq.Add("field", NumericRangeQuery.NewInt64Range("field", 0L, 10L, true, true));
dsr = ds.Search(null, ddq, 10);
Assert.AreEqual(11, dsr.Hits.TotalHits);
Assert.AreEqual("dim=dim path=[] value=11 childCount=2\n b (8)\n a (3)\n", dsr.Facets.GetTopChildren(10, "dim").ToString());
Assert.AreEqual("dim=field path=[] value=21 childCount=5\n less than 10 (10)\n less than or equal to 10 (11)\n over 90 (9)\n 90 or above (10)\n over 1000 (0)\n", dsr.Facets.GetTopChildren(10, "field").ToString());
IOUtils.Dispose(tw, tr, td, w, r, d);
}
public virtual void TestL2OBasic()
{
LabelToOrdinal map = new LabelToOrdinalMap();
CompactLabelToOrdinal compact = new CompactLabelToOrdinal(200, 0.15f, 3);
int n = 50;
string[] uniqueValues = new string[]
{
//@"�",
//@"�r�G��F�\u0382�7\u0019�h�\u0015���#\u001d3\r{��q�_���Ԃ������",
"foo bar one",
//new string(new char[] { (char)65533, (char)65533, (char)65, (char)65533, (char)45, (char)106, (char)40, (char)643, (char)65533, (char)11, (char)65533, (char)88, (char)65533, (char)78, (char)126, (char)56, (char)12, (char)71 }),
//"foo bar two",
//"foo bar three",
//"foo bar four",
//"foo bar five",
//"foo bar six",
//"foo bar seven",
//"foo bar eight",
//"foo bar nine",
//"foo bar ten",
//"foo/bar/one",
//"foo/bar/two",
//"foo/bar/three",
//"foo/bar/four",
//"foo/bar/five",
//"foo/bar/six",
//"foo/bar/seven",
//"foo/bar/eight",
//"foo/bar/nine",
//"foo/bar/ten",
//""
};
var tmpDir = CreateTempDir("testLableToOrdinal");
var f = new FileInfo(Path.Combine(tmpDir.FullName, "CompactLabelToOrdinalTest.tmp"));
int flushInterval = 10;
for (int i = 0; i < n; i++)
{
if (i > 0 && i % flushInterval == 0)
{
using (var fileStream = new FileStream(f.FullName, FileMode.OpenOrCreate, FileAccess.ReadWrite))
{
compact.Flush(fileStream);
}
compact = CompactLabelToOrdinal.Open(f, 0.15f, 3);
//assertTrue(f.Delete());
f.Delete();
assertFalse(File.Exists(f.FullName));
if (flushInterval < (n / 10))
{
flushInterval *= 10;
}
}
FacetLabel label = new FacetLabel();
foreach (string s in uniqueValues)
{
if (s.Length == 0)
{
label = new FacetLabel();
}
else
{
label = new FacetLabel(s.Split(new char[] { '/' }, StringSplitOptions.RemoveEmptyEntries));
}
int ord1 = map.GetOrdinal(label);
int ord2 = compact.GetOrdinal(label);
if (Verbose)
{
Console.WriteLine("Testing label: " + label.ToString());
}
assertEquals(ord1, ord2);
if (ord1 == LabelToOrdinal.INVALID_ORDINAL)
{
ord1 = compact.GetNextOrdinal();
map.AddLabel(label, ord1);
compact.AddLabel(label, ord1);
}
}
}
for (int i = 0; i < uniqueValues.Length; i++)
{
FacetLabel label;
string s = uniqueValues[i];
if (s.Length == 0)
{
label = new FacetLabel();
}
else
{
label = new FacetLabel(s.Split(new char[] { '/' }, StringSplitOptions.RemoveEmptyEntries));
}
int ord1 = map.GetOrdinal(label);
int ord2 = compact.GetOrdinal(label);
if (Verbose)
{
Console.WriteLine("Testing label 2: " + label.ToString());
}
assertEquals(ord1, ord2);
}
}
public virtual void TestL2O()
{
LabelToOrdinal map = new LabelToOrdinalMap();
CompactLabelToOrdinal compact = new CompactLabelToOrdinal(2000000, 0.15f, 3);
int n = AtLeast(10 * 1000);
const int numUniqueValues = 50 * 1000;
string[] uniqueValues = new string[numUniqueValues];
byte[] buffer = new byte[50];
// This is essentially the equivalent of
// CharsetDecoder decoder = StandardCharsets.UTF_8.newDecoder()
// .onUnmappableCharacter(CodingErrorAction.REPLACE)
// .onMalformedInput(CodingErrorAction.REPLACE);
//
// Encoding decoder = Encoding.GetEncoding(Encoding.UTF8.CodePage,
// new EncoderReplacementFallback("?"),
// new DecoderReplacementFallback("?"));
Random random = Random;
for (int i = 0; i < numUniqueValues;)
{
random.NextBytes(buffer);
int size = 1 + random.Next(buffer.Length);
// This test is turning random bytes into a string,
// this is asking for trouble.
Encoding decoder = Encoding.GetEncoding(Encoding.UTF8.CodePage,
new EncoderReplacementFallback("?"),
new DecoderReplacementFallback("?"));
uniqueValues[i] = decoder.GetString(buffer, 0, size);
// we cannot have empty path components, so eliminate all prefix as well
// as middle consecutive delimiter chars.
uniqueValues[i] = Regex.Replace(uniqueValues[i], "/+", "/");
if (uniqueValues[i].StartsWith("/", StringComparison.Ordinal))
{
uniqueValues[i] = uniqueValues[i].Substring(1);
}
if (uniqueValues[i].IndexOf(CompactLabelToOrdinal.TERMINATOR_CHAR) == -1)
{
i++;
}
}
var tmpDir = CreateTempDir("testLableToOrdinal");
var f = new FileInfo(Path.Combine(tmpDir.FullName, "CompactLabelToOrdinalTest.tmp"));
int flushInterval = 10;
for (int i = 0; i < n; i++)
{
if (i > 0 && i % flushInterval == 0)
{
using (var fileStream = new FileStream(f.FullName, FileMode.OpenOrCreate, FileAccess.ReadWrite))
{
compact.Flush(fileStream);
}
compact = CompactLabelToOrdinal.Open(f, 0.15f, 3);
//assertTrue(f.Delete());
f.Delete();
assertFalse(File.Exists(f.FullName));
if (flushInterval < (n / 10))
{
flushInterval *= 10;
}
}
int index = random.Next(numUniqueValues);
FacetLabel label;
string s = uniqueValues[index];
if (s.Length == 0)
{
label = new FacetLabel();
}
else
{
label = new FacetLabel(s.Split(new char[] { '/' }, StringSplitOptions.RemoveEmptyEntries));
}
int ord1 = map.GetOrdinal(label);
int ord2 = compact.GetOrdinal(label);
if (Verbose)
{
Console.WriteLine("Testing label: " + label.ToString());
}
assertEquals(ord1, ord2);
if (ord1 == LabelToOrdinal.INVALID_ORDINAL)
{
ord1 = compact.GetNextOrdinal();
map.AddLabel(label, ord1);
compact.AddLabel(label, ord1);
}
}
for (int i = 0; i < numUniqueValues; i++)
{
FacetLabel label;
string s = uniqueValues[i];
if (s.Length == 0)
{
label = new FacetLabel();
}
else
{
label = new FacetLabel(s.Split(new char[] { '/' }, StringSplitOptions.RemoveEmptyEntries));
}
int ord1 = map.GetOrdinal(label);
int ord2 = compact.GetOrdinal(label);
if (Verbose)
{
Console.WriteLine("Testing label 2: " + label.ToString());
}
assertEquals(ord1, ord2);
}
}
private void Verify(AtomicReader r, int[][] idToOrds, BytesRef[] termsArray, BytesRef prefixRef)
{
DocTermOrds dto = new DocTermOrds(r, r.LiveDocs, "field", prefixRef, int.MaxValue, TestUtil.NextInt32(Random, 2, 10));
FieldCache.Int32s docIDToID = FieldCache.DEFAULT.GetInt32s(r, "id", false);
/*
* for(int docID=0;docID<subR.MaxDoc;docID++) {
* System.out.println(" docID=" + docID + " id=" + docIDToID[docID]);
* }
*/
if (Verbose)
{
Console.WriteLine("TEST: verify prefix=" + (prefixRef == null ? "null" : prefixRef.Utf8ToString()));
Console.WriteLine("TEST: all TERMS:");
TermsEnum allTE = MultiFields.GetTerms(r, "field").GetIterator(null);
int ord = 0;
while (allTE.Next() != null)
{
Console.WriteLine(" ord=" + (ord++) + " term=" + allTE.Term.Utf8ToString());
}
}
//final TermsEnum te = subR.Fields.Terms("field").iterator();
TermsEnum te = dto.GetOrdTermsEnum(r);
if (dto.NumTerms == 0)
{
if (prefixRef == null)
{
Assert.IsNull(MultiFields.GetTerms(r, "field"));
}
else
{
Terms terms = MultiFields.GetTerms(r, "field");
if (terms != null)
{
TermsEnum termsEnum = terms.GetIterator(null);
TermsEnum.SeekStatus result = termsEnum.SeekCeil(prefixRef);
if (result != TermsEnum.SeekStatus.END)
{
Assert.IsFalse(StringHelper.StartsWith(termsEnum.Term, prefixRef), "term=" + termsEnum.Term.Utf8ToString() + " matches prefix=" + prefixRef.Utf8ToString());
}
else
{
// ok
}
}
else
{
// ok
}
}
return;
}
if (Verbose)
{
Console.WriteLine("TEST: TERMS:");
te.SeekExact(0);
while (true)
{
Console.WriteLine(" ord=" + te.Ord + " term=" + te.Term.Utf8ToString());
if (te.Next() == null)
{
break;
}
}
}
SortedSetDocValues iter = dto.GetIterator(r);
for (int docID = 0; docID < r.MaxDoc; docID++)
{
if (Verbose)
{
Console.WriteLine("TEST: docID=" + docID + " of " + r.MaxDoc + " (id=" + docIDToID.Get(docID) + ")");
}
iter.SetDocument(docID);
int[] answers = idToOrds[docIDToID.Get(docID)];
int upto = 0;
long ord;
while ((ord = iter.NextOrd()) != SortedSetDocValues.NO_MORE_ORDS)
{
te.SeekExact(ord);
BytesRef expected = termsArray[answers[upto++]];
if (Verbose)
{
Console.WriteLine(" exp=" + expected.Utf8ToString() + " actual=" + te.Term.Utf8ToString());
}
Assert.AreEqual(expected, te.Term, "expected=" + expected.Utf8ToString() + " actual=" + te.Term.Utf8ToString() + " ord=" + ord);
}
Assert.AreEqual(answers.Length, upto);
}
}
public virtual void TestRandomWithPrefix()
{
Directory dir = NewDirectory();
ISet <string> prefixes = new JCG.HashSet <string>();
int numPrefix = TestUtil.NextInt32(Random, 2, 7);
if (Verbose)
{
Console.WriteLine("TEST: use " + numPrefix + " prefixes");
}
while (prefixes.Count < numPrefix)
{
prefixes.Add(TestUtil.RandomRealisticUnicodeString(Random));
//prefixes.Add(TestUtil.RandomSimpleString(random));
}
string[] prefixesArray = prefixes.ToArray(/*new string[prefixes.Count]*/);
int NUM_TERMS = AtLeast(20);
ISet <BytesRef> terms = new JCG.HashSet <BytesRef>();
while (terms.Count < NUM_TERMS)
{
string s = prefixesArray[Random.Next(prefixesArray.Length)] + TestUtil.RandomRealisticUnicodeString(Random);
//final String s = prefixesArray[random.nextInt(prefixesArray.Length)] + TestUtil.RandomSimpleString(random);
if (s.Length > 0)
{
terms.Add(new BytesRef(s));
}
}
BytesRef[] termsArray = terms.ToArray();
Array.Sort(termsArray);
int NUM_DOCS = AtLeast(100);
IndexWriterConfig conf = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random));
// Sometimes swap in codec that impls ord():
if (Random.Next(10) == 7)
{
Codec codec = TestUtil.AlwaysPostingsFormat(PostingsFormat.ForName("Lucene41WithOrds"));
conf.SetCodec(codec);
}
RandomIndexWriter w = new RandomIndexWriter(Random, dir, conf);
int[][] idToOrds = new int[NUM_DOCS][];
ISet <int?> ordsForDocSet = new JCG.HashSet <int?>();
for (int id = 0; id < NUM_DOCS; id++)
{
Document doc = new Document();
doc.Add(new Int32Field("id", id, Field.Store.NO));
int termCount = TestUtil.NextInt32(Random, 0, 20 * RandomMultiplier);
while (ordsForDocSet.Count < termCount)
{
ordsForDocSet.Add(Random.Next(termsArray.Length));
}
int[] ordsForDoc = new int[termCount];
int upto = 0;
if (Verbose)
{
Console.WriteLine("TEST: doc id=" + id);
}
foreach (int ord in ordsForDocSet)
{
ordsForDoc[upto++] = ord;
Field field = NewStringField("field", termsArray[ord].Utf8ToString(), Field.Store.NO);
if (Verbose)
{
Console.WriteLine(" f=" + termsArray[ord].Utf8ToString());
}
doc.Add(field);
}
ordsForDocSet.Clear();
Array.Sort(ordsForDoc);
idToOrds[id] = ordsForDoc;
w.AddDocument(doc);
}
DirectoryReader r = w.GetReader();
w.Dispose();
if (Verbose)
{
Console.WriteLine("TEST: reader=" + r);
}
AtomicReader slowR = SlowCompositeReaderWrapper.Wrap(r);
foreach (string prefix in prefixesArray)
{
BytesRef prefixRef = prefix == null ? null : new BytesRef(prefix);
int[][] idToOrdsPrefix = new int[NUM_DOCS][];
for (int id = 0; id < NUM_DOCS; id++)
{
int[] docOrds = idToOrds[id];
IList <int?> newOrds = new List <int?>();
foreach (int ord in idToOrds[id])
{
if (StringHelper.StartsWith(termsArray[ord], prefixRef))
{
newOrds.Add(ord);
}
}
int[] newOrdsArray = new int[newOrds.Count];
int upto = 0;
foreach (int ord in newOrds)
{
newOrdsArray[upto++] = ord;
}
idToOrdsPrefix[id] = newOrdsArray;
}
foreach (AtomicReaderContext ctx in r.Leaves)
{
if (Verbose)
{
Console.WriteLine("\nTEST: sub=" + ctx.Reader);
}
Verify((AtomicReader)ctx.Reader, idToOrdsPrefix, termsArray, prefixRef);
}
// Also test top-level reader: its enum does not support
// ord, so this forces the OrdWrapper to run:
if (Verbose)
{
Console.WriteLine("TEST: top reader");
}
Verify(slowR, idToOrdsPrefix, termsArray, prefixRef);
}
FieldCache.DEFAULT.PurgeByCacheKey(slowR.CoreCacheKey);
r.Dispose();
dir.Dispose();
}
public virtual void TestAccquireReleaseRace()
{
DocumentsWriterStallControl ctrl = new DocumentsWriterStallControl();
ctrl.UpdateStalled(false);
AtomicBoolean stop = new AtomicBoolean(false);
AtomicBoolean checkPoint = new AtomicBoolean(true);
int numStallers = AtLeast(1);
int numReleasers = AtLeast(1);
int numWaiters = AtLeast(1);
var sync = new Synchronizer(numStallers + numReleasers, numStallers + numReleasers + numWaiters);
var threads = new ThreadJob[numReleasers + numStallers + numWaiters];
IList <Exception> exceptions = new SynchronizedList <Exception>();
for (int i = 0; i < numReleasers; i++)
{
threads[i] = new Updater(stop, checkPoint, ctrl, sync, true, exceptions);
}
for (int i = numReleasers; i < numReleasers + numStallers; i++)
{
threads[i] = new Updater(stop, checkPoint, ctrl, sync, false, exceptions);
}
for (int i = numReleasers + numStallers; i < numReleasers + numStallers + numWaiters; i++)
{
threads[i] = new Waiter(stop, checkPoint, ctrl, sync, exceptions);
}
Start(threads);
int iters = AtLeast(10000);
float checkPointProbability = TestNightly ? 0.5f : 0.1f;
for (int i = 0; i < iters; i++)
{
if (checkPoint)
{
Assert.IsTrue(sync.updateJoin.Wait(new TimeSpan(0, 0, 0, 10)), "timed out waiting for update threads - deadlock?");
if (exceptions.Count > 0)
{
foreach (Exception throwable in exceptions)
{
Console.WriteLine(throwable.ToString());
Console.Write(throwable.StackTrace);
}
Assert.Fail("got exceptions in threads");
}
if (ctrl.HasBlocked && ctrl.IsHealthy)
{
AssertState(numReleasers, numStallers, numWaiters, threads, ctrl);
}
checkPoint.Value = (false);
sync.waiter.Signal();
sync.leftCheckpoint.Wait();
}
Assert.IsFalse(checkPoint);
Assert.AreEqual(0, sync.waiter.CurrentCount);
if (checkPointProbability >= (float)Random.NextDouble())
{
sync.Reset(numStallers + numReleasers, numStallers + numReleasers + numWaiters);
checkPoint.Value = (true);
}
}
if (!checkPoint)
{
sync.Reset(numStallers + numReleasers, numStallers + numReleasers + numWaiters);
checkPoint.Value = (true);
}
Assert.IsTrue(sync.updateJoin.Wait(new TimeSpan(0, 0, 0, 10)));
AssertState(numReleasers, numStallers, numWaiters, threads, ctrl);
checkPoint.Value = (false);
stop.Value = (true);
sync.waiter.Signal();
sync.leftCheckpoint.Wait();
for (int i = 0; i < threads.Length; i++)
{
ctrl.UpdateStalled(false);
threads[i].Join(2000);
if (threads[i].IsAlive && threads[i] is Waiter)
{
if (threads[i].State == ThreadState.WaitSleepJoin)
{
Assert.Fail("waiter is not released - anyThreadsStalled: " + ctrl.AnyStalledThreads());
}
}
}
}
public virtual void TestNoWaitClose()
{
Directory directory = NewDirectory();
Document doc = new Document();
FieldType customType = new FieldType(TextField.TYPE_STORED);
customType.IsTokenized = false;
Field idField = NewField("id", "", customType);
doc.Add(idField);
for (int pass = 0; pass < 2; pass++)
{
if (Verbose)
{
Console.WriteLine("TEST: pass="******"TEST: iter=" + iter);
}
for (int j = 0; j < 199; j++)
{
idField.SetStringValue(Convert.ToString(iter * 201 + j));
writer.AddDocument(doc);
}
int delID = iter * 199;
for (int j = 0; j < 20; j++)
{
writer.DeleteDocuments(new Term("id", Convert.ToString(delID)));
delID += 5;
}
// Force a bunch of merge threads to kick off so we
// stress out aborting them on close:
((LogMergePolicy)writer.Config.MergePolicy).MergeFactor = 2;
IndexWriter finalWriter = writer;
List <Exception> failure = new List <Exception>();
ThreadJob t1 = new ThreadAnonymousInnerClassHelper(this, doc, finalWriter, failure);
if (failure.Count > 0)
{
throw failure[0];
}
t1.Start();
writer.Dispose(false);
t1.Join();
// Make sure reader can read
IndexReader reader = DirectoryReader.Open(directory);
reader.Dispose();
// Reopen
writer = new IndexWriter(directory, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random)).SetOpenMode(OpenMode.APPEND).SetMergePolicy(NewLogMergePolicy()));
}
writer.Dispose();
}
directory.Dispose();
}
public virtual void TestRandomDoubles()
{
Directory dir = NewDirectory();
RandomIndexWriter w = new RandomIndexWriter(
#if FEATURE_INSTANCE_TESTDATA_INITIALIZATION
this,
#endif
Random, dir);
int numDocs = AtLeast(1000);
double[] values = new double[numDocs];
double minValue = double.PositiveInfinity;
double maxValue = double.NegativeInfinity;
for (int i = 0; i < numDocs; i++)
{
Document doc = new Document();
double v = Random.NextDouble();
values[i] = v;
doc.Add(new DoubleDocValuesField("field", v));
doc.Add(new DoubleField("field", v, Field.Store.NO));
w.AddDocument(doc);
minValue = Math.Min(minValue, v);
maxValue = Math.Max(maxValue, v);
}
IndexReader r = w.GetReader();
IndexSearcher s = NewSearcher(r);
FacetsConfig config = new FacetsConfig();
int numIters = AtLeast(10);
for (int iter = 0; iter < numIters; iter++)
{
if (VERBOSE)
{
Console.WriteLine("TEST: iter=" + iter);
}
int numRange = TestUtil.NextInt32(Random, 1, 5);
DoubleRange[] ranges = new DoubleRange[numRange];
int[] expectedCounts = new int[numRange];
double minAcceptedValue = double.PositiveInfinity;
double maxAcceptedValue = double.NegativeInfinity;
for (int rangeID = 0; rangeID < numRange; rangeID++)
{
double min;
if (rangeID > 0 && Random.Next(10) == 7)
{
// Use an existing boundary:
DoubleRange prevRange = ranges[Random.Next(rangeID)];
if (Random.NextBoolean())
{
min = prevRange.Min;
}
else
{
min = prevRange.Max;
}
}
else
{
min = Random.NextDouble();
}
double max;
if (rangeID > 0 && Random.Next(10) == 7)
{
// Use an existing boundary:
DoubleRange prevRange = ranges[Random.Next(rangeID)];
if (Random.NextBoolean())
{
max = prevRange.Min;
}
else
{
max = prevRange.Max;
}
}
else
{
max = Random.NextDouble();
}
if (min > max)
{
double x = min;
min = max;
max = x;
}
bool minIncl;
bool maxIncl;
if (min == max)
{
minIncl = true;
maxIncl = true;
}
else
{
minIncl = Random.NextBoolean();
maxIncl = Random.NextBoolean();
}
ranges[rangeID] = new DoubleRange("r" + rangeID, min, minIncl, max, maxIncl);
// Do "slow but hopefully correct" computation of
// expected count:
for (int i = 0; i < numDocs; i++)
{
bool accept = true;
if (minIncl)
{
accept &= values[i] >= min;
}
else
{
accept &= values[i] > min;
}
if (maxIncl)
{
accept &= values[i] <= max;
}
else
{
accept &= values[i] < max;
}
if (accept)
{
expectedCounts[rangeID]++;
minAcceptedValue = Math.Min(minAcceptedValue, values[i]);
maxAcceptedValue = Math.Max(maxAcceptedValue, values[i]);
}
}
}
FacetsCollector sfc = new FacetsCollector();
s.Search(new MatchAllDocsQuery(), sfc);
Filter fastMatchFilter;
if (Random.NextBoolean())
{
if (Random.NextBoolean())
{
fastMatchFilter = NumericRangeFilter.NewDoubleRange("field", minValue, maxValue, true, true);
}
else
{
fastMatchFilter = NumericRangeFilter.NewDoubleRange("field", minAcceptedValue, maxAcceptedValue, true, true);
}
}
else
{
fastMatchFilter = null;
}
ValueSource vs = new DoubleFieldSource("field");
Facets facets = new DoubleRangeFacetCounts("field", vs, sfc, fastMatchFilter, ranges);
FacetResult result = facets.GetTopChildren(10, "field");
Assert.AreEqual(numRange, result.LabelValues.Length);
for (int rangeID = 0; rangeID < numRange; rangeID++)
{
if (VERBOSE)
{
Console.WriteLine(" range " + rangeID + " expectedCount=" + expectedCounts[rangeID]);
}
LabelAndValue subNode = result.LabelValues[rangeID];
Assert.AreEqual("r" + rangeID, subNode.Label);
Assert.AreEqual(expectedCounts[rangeID], (int)subNode.Value);
DoubleRange range = ranges[rangeID];
// Test drill-down:
DrillDownQuery ddq = new DrillDownQuery(config);
if (Random.NextBoolean())
{
if (Random.NextBoolean())
{
ddq.Add("field", NumericRangeFilter.NewDoubleRange("field", range.Min, range.Max, range.MinInclusive, range.MaxInclusive));
}
else
{
ddq.Add("field", NumericRangeQuery.NewDoubleRange("field", range.Min, range.Max, range.MinInclusive, range.MaxInclusive));
}
}
else
{
ddq.Add("field", range.GetFilter(fastMatchFilter, vs));
}
Assert.AreEqual(expectedCounts[rangeID], s.Search(ddq, 10).TotalHits);
}
}
IOUtils.Dispose(w, r, dir);
}
public virtual void TestConcurrency()
{
int ncats = AtLeast(100000); // add many categories
int range = ncats * 3; // affects the categories selection
AtomicInt32 numCats = new AtomicInt32(ncats);
Directory dir = NewDirectory();
var values = new ConcurrentDictionary <string, string>();
double d = Random.NextDouble();
ITaxonomyWriterCache cache;
if (d < 0.7)
{
// this is the fastest, yet most memory consuming
cache = new Cl2oTaxonomyWriterCache(1024, 0.15f, 3);
}
// LUCENENET specific - this option takes too long to get under the 1 hour job limit in Azure DevOps
//else if (TestNightly && d > 0.98)
//{
// // this is the slowest, but tests the writer concurrency when no caching is done.
// // only pick it during NIGHTLY tests, and even then, with very low chances.
// cache = NO_OP_CACHE;
//}
else
{
// this is slower than CL2O, but less memory consuming, and exercises finding categories on disk too.
cache = new LruTaxonomyWriterCache(ncats / 10);
}
if (Verbose)
{
Console.WriteLine("TEST: use cache=" + cache);
}
var tw = new DirectoryTaxonomyWriter(dir, OpenMode.CREATE, cache);
ThreadJob[] addThreads = new ThreadJob[AtLeast(4)];
for (int z = 0; z < addThreads.Length; z++)
{
addThreads[z] = new ThreadAnonymousClass(range, numCats, values, tw);
}
foreach (var t in addThreads)
{
t.Start();
}
foreach (var t in addThreads)
{
t.Join();
}
tw.Dispose();
DirectoryTaxonomyReader dtr = new DirectoryTaxonomyReader(dir);
// +1 for root category
if (values.Count + 1 != dtr.Count)
{
foreach (string value in values.Keys)
{
FacetLabel label = new FacetLabel(FacetsConfig.StringToPath(value));
if (dtr.GetOrdinal(label) == -1)
{
Console.WriteLine("FAIL: path=" + label + " not recognized");
}
}
fail("mismatch number of categories");
}
int[] parents = dtr.ParallelTaxonomyArrays.Parents;
foreach (string cat in values.Keys)
{
FacetLabel cp = new FacetLabel(FacetsConfig.StringToPath(cat));
Assert.IsTrue(dtr.GetOrdinal(cp) > 0, "category not found " + cp);
int level = cp.Length;
int parentOrd = 0; // for root, parent is always virtual ROOT (ord=0)
FacetLabel path /*= new FacetLabel()*/;
for (int i = 0; i < level; i++)
{
path = cp.Subpath(i + 1);
int ord = dtr.GetOrdinal(path);
Assert.AreEqual(parentOrd, parents[ord], "invalid parent for cp=" + path);
parentOrd = ord; // next level should have this parent
}
}
IOUtils.Dispose(dtr, dir);
}
public virtual void TestNrt()
{
Store.Directory dir = NewDirectory();
Store.Directory taxoDir = NewDirectory();
IndexWriterConfig iwc = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random));
// Don't allow tiny maxBufferedDocs; it can make this
// test too slow:
iwc.SetMaxBufferedDocs(Math.Max(500, iwc.MaxBufferedDocs));
// MockRandom/AlcololicMergePolicy are too slow:
TieredMergePolicy tmp = new TieredMergePolicy();
tmp.FloorSegmentMB = .001;
iwc.SetMergePolicy(tmp);
IndexWriter w = new IndexWriter(dir, iwc);
var tw = new DirectoryTaxonomyWriter(taxoDir);
FacetsConfig config = new FacetsConfig();
config.SetMultiValued("field", true);
AtomicBoolean stop = new AtomicBoolean();
// How many unique facets to index before stopping:
//int ordLimit = TestNightly ? 100000 : 6000;
// LUCENENET specific: 100000 facets takes about 2-3 hours. To keep it under
// the 1 hour free limit of Azure DevOps, this was reduced to 30000.
int ordLimit = TestNightly ? 30000 : 6000;
var indexer = new IndexerThread(w, config, tw, null, ordLimit, stop);
var mgr = new SearcherTaxonomyManager(w, true, null, tw);
var reopener = new ThreadAnonymousInnerClassHelper(stop, mgr);
reopener.Name = "reopener";
reopener.Start();
indexer.Name = "indexer";
indexer.Start();
try
{
while (!stop)
{
SearcherAndTaxonomy pair = mgr.Acquire();
try
{
//System.out.println("search maxOrd=" + pair.taxonomyReader.getSize());
FacetsCollector sfc = new FacetsCollector();
pair.Searcher.Search(new MatchAllDocsQuery(), sfc);
Facets facets = GetTaxonomyFacetCounts(pair.TaxonomyReader, config, sfc);
FacetResult result = facets.GetTopChildren(10, "field");
if (pair.Searcher.IndexReader.NumDocs > 0)
{
//System.out.println(pair.taxonomyReader.getSize());
Assert.IsTrue(result.ChildCount > 0);
Assert.IsTrue(result.LabelValues.Length > 0);
}
//if (VERBOSE) {
//System.out.println("TEST: facets=" + FacetTestUtils.toString(results.get(0)));
//}
}
finally
{
mgr.Release(pair);
}
}
}
finally
{
indexer.Join();
reopener.Join();
}
if (Verbose)
{
Console.WriteLine("TEST: now stop");
}
IOUtils.Dispose(mgr, tw, w, taxoDir, dir);
}
public virtual void Test()
{
Random random = new Random(Random.Next());
LineFileDocs docs = new LineFileDocs(random, DefaultCodecSupportsDocValues);
Directory d = NewDirectory();
MockAnalyzer analyzer = new MockAnalyzer(LuceneTestCase.Random);
analyzer.MaxTokenLength = TestUtil.NextInt32(LuceneTestCase.Random, 1, IndexWriter.MAX_TERM_LENGTH);
RandomIndexWriter w = new RandomIndexWriter(
#if FEATURE_INSTANCE_TESTDATA_INITIALIZATION
this,
#endif
LuceneTestCase.Random, d, analyzer);
int numDocs = AtLeast(10);
for (int docCount = 0; docCount < numDocs; docCount++)
{
w.AddDocument(docs.NextDoc());
}
IndexReader r = w.GetReader();
w.Dispose();
List <BytesRef> terms = new List <BytesRef>();
TermsEnum termsEnum = MultiFields.GetTerms(r, "body").GetIterator(null);
BytesRef term;
while ((term = termsEnum.Next()) != null)
{
terms.Add(BytesRef.DeepCopyOf(term));
}
if (VERBOSE)
{
Console.WriteLine("TEST: " + terms.Count + " terms");
}
int upto = -1;
int iters = AtLeast(200);
for (int iter = 0; iter < iters; iter++)
{
bool isEnd;
if (upto != -1 && LuceneTestCase.Random.NextBoolean())
{
// next
if (VERBOSE)
{
Console.WriteLine("TEST: iter next");
}
isEnd = termsEnum.Next() == null;
upto++;
if (isEnd)
{
if (VERBOSE)
{
Console.WriteLine(" end");
}
Assert.AreEqual(upto, terms.Count);
upto = -1;
}
else
{
if (VERBOSE)
{
Console.WriteLine(" got term=" + termsEnum.Term.Utf8ToString() + " expected=" + terms[upto].Utf8ToString());
}
Assert.IsTrue(upto < terms.Count);
Assert.AreEqual(terms[upto], termsEnum.Term);
}
}
else
{
BytesRef target;
string exists;
if (LuceneTestCase.Random.NextBoolean())
{
// likely fake term
if (LuceneTestCase.Random.NextBoolean())
{
target = new BytesRef(TestUtil.RandomSimpleString(LuceneTestCase.Random));
}
else
{
target = new BytesRef(TestUtil.RandomRealisticUnicodeString(LuceneTestCase.Random));
}
exists = "likely not";
}
else
{
// real term
target = terms[LuceneTestCase.Random.Next(terms.Count)];
exists = "yes";
}
upto = terms.BinarySearch(target);
if (LuceneTestCase.Random.NextBoolean())
{
if (VERBOSE)
{
Console.WriteLine("TEST: iter seekCeil target=" + target.Utf8ToString() + " exists=" + exists);
}
// seekCeil
TermsEnum.SeekStatus status = termsEnum.SeekCeil(target);
if (VERBOSE)
{
Console.WriteLine(" got " + status);
}
if (upto < 0)
{
upto = -(upto + 1);
if (upto >= terms.Count)
{
Assert.AreEqual(TermsEnum.SeekStatus.END, status);
upto = -1;
}
else
{
Assert.AreEqual(TermsEnum.SeekStatus.NOT_FOUND, status);
Assert.AreEqual(terms[upto], termsEnum.Term);
}
}
else
{
Assert.AreEqual(TermsEnum.SeekStatus.FOUND, status);
Assert.AreEqual(terms[upto], termsEnum.Term);
}
}
else
{
if (VERBOSE)
{
Console.WriteLine("TEST: iter seekExact target=" + target.Utf8ToString() + " exists=" + exists);
}
// seekExact
bool result = termsEnum.SeekExact(target);
if (VERBOSE)
{
Console.WriteLine(" got " + result);
}
if (upto < 0)
{
Assert.IsFalse(result);
upto = -1;
}
else
{
Assert.IsTrue(result);
Assert.AreEqual(target, termsEnum.Term);
}
}
}
}
r.Dispose();
d.Dispose();
docs.Dispose();
}
public virtual void Test_Directory() // LUCENENET specific - name collides with property of LuceneTestCase
{
Store.Directory indexDir = NewDirectory();
Store.Directory taxoDir = NewDirectory();
IndexWriter w = new IndexWriter(indexDir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random)));
var tw = new DirectoryTaxonomyWriter(taxoDir);
// first empty commit
w.Commit();
tw.Commit();
var mgr = new SearcherTaxonomyManager(indexDir, taxoDir, null);
FacetsConfig config = new FacetsConfig();
config.SetMultiValued("field", true);
AtomicBoolean stop = new AtomicBoolean();
// How many unique facets to index before stopping:
//int ordLimit = TestNightly ? 100000 : 6000;
// LUCENENET specific: 100000 facets takes about 2-3 hours. To keep it under
// the 1 hour free limit of Azure DevOps, this was reduced to 30000.
int ordLimit = TestNightly ? 30000 : 6000;
var indexer = new IndexerThread(w, config, tw, mgr, ordLimit, stop);
indexer.Start();
try
{
while (!stop)
{
SearcherAndTaxonomy pair = mgr.Acquire();
try
{
//System.out.println("search maxOrd=" + pair.taxonomyReader.getSize());
FacetsCollector sfc = new FacetsCollector();
pair.Searcher.Search(new MatchAllDocsQuery(), sfc);
Facets facets = GetTaxonomyFacetCounts(pair.TaxonomyReader, config, sfc);
FacetResult result = facets.GetTopChildren(10, "field");
if (pair.Searcher.IndexReader.NumDocs > 0)
{
//System.out.println(pair.taxonomyReader.getSize());
Assert.IsTrue(result.ChildCount > 0);
Assert.IsTrue(result.LabelValues.Length > 0);
}
//if (VERBOSE) {
//System.out.println("TEST: facets=" + FacetTestUtils.toString(results.get(0)));
//}
}
finally
{
mgr.Release(pair);
}
}
}
finally
{
indexer.Join();
}
if (Verbose)
{
Console.WriteLine("TEST: now stop");
}
IOUtils.Dispose(mgr, tw, w, taxoDir, indexDir);
}
private void Verify(BytesStore bytes, byte[] expected, int totalLength)
{
Assert.AreEqual(totalLength, bytes.Position);
if (totalLength == 0)
{
return;
}
if (VERBOSE)
{
Console.WriteLine(" verify...");
}
// First verify whole thing in one blast:
byte[] actual = new byte[totalLength];
if (Random.NextBoolean())
{
if (VERBOSE)
{
Console.WriteLine(" bulk: reversed");
}
// reversed
FST.BytesReader r2 = bytes.GetReverseReader();
Assert.IsTrue(r2.IsReversed);
r2.Position = totalLength - 1;
r2.ReadBytes(actual, 0, actual.Length);
int start = 0;
int end = totalLength - 1;
while (start < end)
{
byte b = actual[start];
actual[start] = actual[end];
actual[end] = b;
start++;
end--;
}
}
else
{
// forward
if (VERBOSE)
{
Console.WriteLine(" bulk: forward");
}
FST.BytesReader r3 = bytes.GetForwardReader();
Assert.IsFalse(r3.IsReversed);
r3.ReadBytes(actual, 0, actual.Length);
}
for (int i = 0; i < totalLength; i++)
{
assertEquals("byte @ index=" + i, expected[i], actual[i]);
}
FST.BytesReader r;
// Then verify ops:
bool reversed = Random.NextBoolean();
if (reversed)
{
if (VERBOSE)
{
Console.WriteLine(" ops: reversed");
}
r = bytes.GetReverseReader();
}
else
{
if (VERBOSE)
{
Console.WriteLine(" ops: forward");
}
r = bytes.GetForwardReader();
}
if (totalLength > 1)
{
int numOps = TestUtil.NextInt32(Random, 100, 200);
for (int op = 0; op < numOps; op++)
{
int numBytes = Random.Next(Math.Min(1000, totalLength - 1));
int pos;
if (reversed)
{
pos = TestUtil.NextInt32(Random, numBytes, totalLength - 1);
}
else
{
pos = Random.Next(totalLength - numBytes);
}
if (VERBOSE)
{
Console.WriteLine(" op iter=" + op + " reversed=" + reversed + " numBytes=" + numBytes + " pos=" + pos);
}
byte[] temp = new byte[numBytes];
r.Position = pos;
Assert.AreEqual(pos, r.Position);
r.ReadBytes(temp, 0, temp.Length);
for (int i = 0; i < numBytes; i++)
{
byte expectedByte;
if (reversed)
{
expectedByte = expected[pos - i];
}
else
{
expectedByte = expected[pos + i];
}
assertEquals("byte @ index=" + i, expectedByte, temp[i]);
}
int left;
int expectedPos;
if (reversed)
{
expectedPos = pos - numBytes;
left = (int)r.Position;
}
else
{
expectedPos = pos + numBytes;
left = (int)(totalLength - r.Position);
}
Assert.AreEqual(expectedPos, r.Position);
if (left > 4)
{
int skipBytes = Random.Next(left - 4);
int expectedInt = 0;
if (reversed)
{
expectedPos -= skipBytes;
expectedInt |= (expected[expectedPos--] & 0xFF) << 24;
expectedInt |= (expected[expectedPos--] & 0xFF) << 16;
expectedInt |= (expected[expectedPos--] & 0xFF) << 8;
expectedInt |= (expected[expectedPos--] & 0xFF);
}
else
{
expectedPos += skipBytes;
expectedInt |= (expected[expectedPos++] & 0xFF) << 24;
expectedInt |= (expected[expectedPos++] & 0xFF) << 16;
expectedInt |= (expected[expectedPos++] & 0xFF) << 8;
expectedInt |= (expected[expectedPos++] & 0xFF);
}
if (VERBOSE)
{
Console.WriteLine(" skip numBytes=" + skipBytes);
Console.WriteLine(" readInt");
}
r.SkipBytes(skipBytes);
Assert.AreEqual(expectedInt, r.ReadInt32());
}
}
}
}
public override void Run()
{
try
{
var seen = new JCG.HashSet <string>();
IList <string> paths = new List <string>();
while (true)
{
Document doc = new Document();
int numPaths = TestUtil.NextInt32(Random, 1, 5);
for (int i = 0; i < numPaths; i++)
{
string path;
if (paths.Count > 0 && Random.Next(5) != 4)
{
// Use previous path
path = paths[Random.Next(paths.Count)];
}
else
{
// Create new path
path = null;
while (true)
{
path = TestUtil.RandomRealisticUnicodeString(Random);
if (path.Length != 0 && !seen.Contains(path))
{
seen.Add(path);
paths.Add(path);
break;
}
}
}
doc.Add(new FacetField("field", path));
}
try
{
w.AddDocument(config.Build(tw, doc));
if (mgr != null && Random.NextDouble() < 0.02)
{
w.Commit();
tw.Commit();
mgr.MaybeRefresh();
}
}
catch (IOException ioe)
{
throw new Exception(ioe.ToString(), ioe);
}
if (Verbose)
{
Console.WriteLine("TW size=" + tw.Count + " vs " + ordLimit);
}
if (tw.Count >= ordLimit)
{
break;
}
}
}
finally
{
stop.Value = true;
}
}
public virtual void TestRandomStringSort()
{
Random random = new Random(Random.Next());
int NUM_DOCS = AtLeast(100);
Directory dir = NewDirectory();
RandomIndexWriter writer = new RandomIndexWriter(
#if FEATURE_INSTANCE_TESTDATA_INITIALIZATION
this,
#endif
random, dir);
bool allowDups = random.NextBoolean();
ISet <string> seen = new JCG.HashSet <string>();
int maxLength = TestUtil.NextInt32(random, 5, 100);
if (Verbose)
{
Console.WriteLine("TEST: NUM_DOCS=" + NUM_DOCS + " maxLength=" + maxLength + " allowDups=" + allowDups);
}
int numDocs = 0;
IList <BytesRef> docValues = new List <BytesRef>();
// TODO: deletions
while (numDocs < NUM_DOCS)
{
Document doc = new Document();
// 10% of the time, the document is missing the value:
BytesRef br;
if (LuceneTestCase.Random.Next(10) != 7)
{
string s;
if (random.NextBoolean())
{
s = TestUtil.RandomSimpleString(random, maxLength);
}
else
{
s = TestUtil.RandomUnicodeString(random, maxLength);
}
if (!allowDups)
{
if (seen.Contains(s))
{
continue;
}
seen.Add(s);
}
if (Verbose)
{
Console.WriteLine(" " + numDocs + ": s=" + s);
}
br = new BytesRef(s);
if (DefaultCodecSupportsDocValues)
{
doc.Add(new SortedDocValuesField("stringdv", br));
doc.Add(new NumericDocValuesField("id", numDocs));
}
else
{
doc.Add(NewStringField("id", Convert.ToString(numDocs), Field.Store.NO));
}
doc.Add(NewStringField("string", s, Field.Store.NO));
docValues.Add(br);
}
else
{
br = null;
if (Verbose)
{
Console.WriteLine(" " + numDocs + ": <missing>");
}
docValues.Add(null);
if (DefaultCodecSupportsDocValues)
{
doc.Add(new NumericDocValuesField("id", numDocs));
}
else
{
doc.Add(NewStringField("id", Convert.ToString(numDocs), Field.Store.NO));
}
}
doc.Add(new StoredField("id", numDocs));
writer.AddDocument(doc);
numDocs++;
if (random.Next(40) == 17)
{
// force flush
writer.GetReader().Dispose();
}
}
IndexReader r = writer.GetReader();
writer.Dispose();
if (Verbose)
{
Console.WriteLine(" reader=" + r);
}
IndexSearcher idxS = NewSearcher(
#if FEATURE_INSTANCE_TESTDATA_INITIALIZATION
this,
#endif
r, false);
int ITERS = AtLeast(100);
for (int iter = 0; iter < ITERS; iter++)
{
bool reverse = random.NextBoolean();
TopFieldDocs hits;
SortField sf;
bool sortMissingLast;
bool missingIsNull;
if (DefaultCodecSupportsDocValues && random.NextBoolean())
{
sf = new SortField("stringdv", SortFieldType.STRING, reverse);
// Can only use sort missing if the DVFormat
// supports docsWithField:
sortMissingLast = DefaultCodecSupportsDocsWithField && Random.NextBoolean();
missingIsNull = DefaultCodecSupportsDocsWithField;
}
else
{
sf = new SortField("string", SortFieldType.STRING, reverse);
sortMissingLast = Random.NextBoolean();
missingIsNull = true;
}
if (sortMissingLast)
{
sf.MissingValue = SortField.STRING_LAST;
}
Sort sort;
if (random.NextBoolean())
{
sort = new Sort(sf);
}
else
{
sort = new Sort(sf, SortField.FIELD_DOC);
}
int hitCount = TestUtil.NextInt32(random, 1, r.MaxDoc + 20);
RandomFilter f = new RandomFilter(random, (float)random.NextDouble(), docValues);
int queryType = random.Next(3);
if (queryType == 0)
{
// force out of order
BooleanQuery bq = new BooleanQuery();
// Add a Query with SHOULD, since bw.Scorer() returns BooleanScorer2
// which delegates to BS if there are no mandatory clauses.
bq.Add(new MatchAllDocsQuery(), Occur.SHOULD);
// Set minNrShouldMatch to 1 so that BQ will not optimize rewrite to return
// the clause instead of BQ.
bq.MinimumNumberShouldMatch = 1;
hits = idxS.Search(bq, f, hitCount, sort, random.NextBoolean(), random.NextBoolean());
}
else if (queryType == 1)
{
hits = idxS.Search(new ConstantScoreQuery(f), null, hitCount, sort, random.NextBoolean(), random.NextBoolean());
}
else
{
hits = idxS.Search(new MatchAllDocsQuery(), f, hitCount, sort, random.NextBoolean(), random.NextBoolean());
}
if (Verbose)
{
Console.WriteLine("\nTEST: iter=" + iter + " " + hits.TotalHits + " hits; topN=" + hitCount + "; reverse=" + reverse + "; sortMissingLast=" + sortMissingLast + " sort=" + sort);
}
// Compute expected results:
var expected = f.matchValues.ToList();
expected.Sort(Comparer <BytesRef> .Create((a, b) =>
{
if (a == null)
{
if (b == null)
{
return(0);
}
if (sortMissingLast)
{
return(1);
}
else
{
return(-1);
}
}
else if (b == null)
{
if (sortMissingLast)
{
return(-1);
}
else
{
return(1);
}
}
else
{
return(a.CompareTo(b));
}
}));
if (reverse)
{
expected.Reverse();
}
if (Verbose)
{
Console.WriteLine(" expected:");
for (int idx = 0; idx < expected.Count; idx++)
{
BytesRef br = expected[idx];
if (br == null && missingIsNull == false)
{
br = new BytesRef();
}
Console.WriteLine(" " + idx + ": " + (br == null ? "<missing>" : br.Utf8ToString()));
if (idx == hitCount - 1)
{
break;
}
}
}
if (Verbose)
{
Console.WriteLine(" actual:");
for (int hitIDX = 0; hitIDX < hits.ScoreDocs.Length; hitIDX++)
{
FieldDoc fd = (FieldDoc)hits.ScoreDocs[hitIDX];
BytesRef br = (BytesRef)fd.Fields[0];
Console.WriteLine(" " + hitIDX + ": " + (br == null ? "<missing>" : br.Utf8ToString()) + " id=" + idxS.Doc(fd.Doc).Get("id"));
}
}
for (int hitIDX = 0; hitIDX < hits.ScoreDocs.Length; hitIDX++)
{
FieldDoc fd = (FieldDoc)hits.ScoreDocs[hitIDX];
BytesRef br = expected[hitIDX];
if (br == null && missingIsNull == false)
{
br = new BytesRef();
}
// Normally, the old codecs (that don't support
// docsWithField via doc values) will always return
// an empty BytesRef for the missing case; however,
// if all docs in a given segment were missing, in
// that case it will return null! So we must map
// null here, too:
BytesRef br2 = (BytesRef)fd.Fields[0];
if (br2 == null && missingIsNull == false)
{
br2 = new BytesRef();
}
Assert.AreEqual(br, br2, "hit=" + hitIDX + " has wrong sort value");
}
}
r.Dispose();
dir.Dispose();
}
public override void Run()
{
try
{
while (Operations > 0)
{
int oper = rand.Next(100);
if (oper < CommitPercent)
{
if (NumCommitting.IncrementAndGet() <= MaxConcurrentCommits)
{
IDictionary <int, long> newCommittedModel;
long version;
DirectoryReader oldReader;
lock (OuterInstance)
{
newCommittedModel = new Dictionary <int, long>(OuterInstance.Model); // take a snapshot
version = OuterInstance.SnapshotCount++;
oldReader = OuterInstance.Reader;
oldReader.IncRef(); // increment the reference since we will use this for reopening
}
DirectoryReader newReader;
if (rand.Next(100) < SoftCommitPercent)
{
// assertU(h.Commit("softCommit","true"));
if (Random.NextBoolean())
{
if (VERBOSE)
{
Console.WriteLine("TEST: " + Thread.CurrentThread.Name + ": call writer.getReader");
}
newReader = Writer.GetReader(true);
}
else
{
if (VERBOSE)
{
Console.WriteLine("TEST: " + Thread.CurrentThread.Name + ": reopen reader=" + oldReader + " version=" + version);
}
newReader = DirectoryReader.OpenIfChanged(oldReader, Writer.IndexWriter, true);
}
}
else
{
// assertU(commit());
if (VERBOSE)
{
Console.WriteLine("TEST: " + Thread.CurrentThread.Name + ": commit+reopen reader=" + oldReader + " version=" + version);
}
Writer.Commit();
if (VERBOSE)
{
Console.WriteLine("TEST: " + Thread.CurrentThread.Name + ": now reopen after commit");
}
newReader = DirectoryReader.OpenIfChanged(oldReader);
}
// Code below assumes newReader comes w/
// extra ref:
if (newReader == null)
{
oldReader.IncRef();
newReader = oldReader;
}
oldReader.DecRef();
lock (OuterInstance)
{
// install the new reader if it's newest (and check the current version since another reader may have already been installed)
//System.out.println(Thread.currentThread().getName() + ": newVersion=" + newReader.getVersion());
Debug.Assert(newReader.RefCount > 0);
Debug.Assert(OuterInstance.Reader.RefCount > 0);
if (newReader.Version > OuterInstance.Reader.Version)
{
if (VERBOSE)
{
Console.WriteLine("TEST: " + Thread.CurrentThread.Name + ": install new reader=" + newReader);
}
OuterInstance.Reader.DecRef();
OuterInstance.Reader = newReader;
// Silly: forces fieldInfos to be
// loaded so we don't hit IOE on later
// reader.toString
newReader.ToString();
// install this snapshot only if it's newer than the current one
if (version >= OuterInstance.CommittedModelClock)
{
if (VERBOSE)
{
Console.WriteLine("TEST: " + Thread.CurrentThread.Name + ": install new model version=" + version);
}
OuterInstance.CommittedModel = newCommittedModel;
OuterInstance.CommittedModelClock = version;
}
else
{
if (VERBOSE)
{
Console.WriteLine("TEST: " + Thread.CurrentThread.Name + ": skip install new model version=" + version);
}
}
}
else
{
// if the same reader, don't decRef.
if (VERBOSE)
{
Console.WriteLine("TEST: " + Thread.CurrentThread.Name + ": skip install new reader=" + newReader);
}
newReader.DecRef();
}
}
}
NumCommitting.DecrementAndGet();
}
else
{
int id = rand.Next(Ndocs);
object sync = OuterInstance.SyncArr[id];
// set the lastId before we actually change it sometimes to try and
// uncover more race conditions between writing and reading
bool before = Random.NextBoolean();
if (before)
{
OuterInstance.LastId = id;
}
// We can't concurrently update the same document and retain our invariants of increasing values
// since we can't guarantee what order the updates will be executed.
lock (sync)
{
long val = OuterInstance.Model[id];
long nextVal = Math.Abs(val) + 1;
if (oper < CommitPercent + DeletePercent)
{
// assertU("<delete><id>" + id + "</id></delete>");
// add tombstone first
if (Tombstones)
{
Document d = new Document();
d.Add(NewStringField("id", "-" + Convert.ToString(id), Documents.Field.Store.YES));
d.Add(NewField(OuterInstance.Field, Convert.ToString(nextVal), StoredOnlyType));
Writer.UpdateDocument(new Term("id", "-" + Convert.ToString(id)), d);
}
if (VERBOSE)
{
Console.WriteLine("TEST: " + Thread.CurrentThread.Name + ": term delDocs id:" + id + " nextVal=" + nextVal);
}
Writer.DeleteDocuments(new Term("id", Convert.ToString(id)));
OuterInstance.Model[id] = -nextVal;
}
else if (oper < CommitPercent + DeletePercent + DeleteByQueryPercent)
{
//assertU("<delete><query>id:" + id + "</query></delete>");
// add tombstone first
if (Tombstones)
{
Document d = new Document();
d.Add(NewStringField("id", "-" + Convert.ToString(id), Documents.Field.Store.YES));
d.Add(NewField(OuterInstance.Field, Convert.ToString(nextVal), StoredOnlyType));
Writer.UpdateDocument(new Term("id", "-" + Convert.ToString(id)), d);
}
if (VERBOSE)
{
Console.WriteLine("TEST: " + Thread.CurrentThread.Name + ": query delDocs id:" + id + " nextVal=" + nextVal);
}
Writer.DeleteDocuments(new TermQuery(new Term("id", Convert.ToString(id))));
OuterInstance.Model[id] = -nextVal;
}
else
{
// assertU(adoc("id",Integer.toString(id), field, Long.toString(nextVal)));
Document d = new Document();
d.Add(NewStringField("id", Convert.ToString(id), Documents.Field.Store.YES));
d.Add(NewField(OuterInstance.Field, Convert.ToString(nextVal), StoredOnlyType));
if (VERBOSE)
{
Console.WriteLine("TEST: " + Thread.CurrentThread.Name + ": u id:" + id + " val=" + nextVal);
}
Writer.UpdateDocument(new Term("id", Convert.ToString(id)), d);
if (Tombstones)
{
// remove tombstone after new addition (this should be optional?)
Writer.DeleteDocuments(new Term("id", "-" + Convert.ToString(id)));
}
OuterInstance.Model[id] = nextVal;
}
}
if (!before)
{
OuterInstance.LastId = id;
}
}
}
}
catch (Exception e)
{
Console.WriteLine(Thread.CurrentThread.Name + ": FAILED: unexpected exception");
Console.WriteLine(e.StackTrace);
throw new Exception(e.Message, e);
}
}
public override void Run()
{
try
{
IDictionary <string, int?> values = new Dictionary <string, int?>();
IList <string> allIDs = new SynchronizedList <string>();
startingGun.Wait();
for (int iter = 0; iter < iters; iter++)
{
// Add/update a document
Document doc = new Document();
// Threads must not update the same id at the
// same time:
if (threadRandom.NextDouble() <= addChance)
{
string id = string.Format(CultureInfo.InvariantCulture, "{0}_{1:X4}", threadID, threadRandom.Next(idCount));
int field = threadRandom.Next(int.MaxValue);
doc.Add(new StringField("id", id, Field.Store.YES));
doc.Add(new Int32Field("field", (int)field, Field.Store.YES));
w.UpdateDocument(new Term("id", id), doc);
rt.Add(id, field);
if (!values.ContainsKey(id))//Key didn't exist before
{
allIDs.Add(id);
}
values[id] = field;
}
if (allIDs.Count > 0 && threadRandom.NextDouble() <= deleteChance)
{
string randomID = allIDs[threadRandom.Next(allIDs.Count)];
w.DeleteDocuments(new Term("id", randomID));
rt.Delete(randomID);
values[randomID] = missing;
}
if (threadRandom.NextDouble() <= reopenChance || rt.Count > 10000)
{
//System.out.println("refresh @ " + rt.Size());
mgr.MaybeRefresh();
if (Verbose)
{
IndexSearcher s = mgr.Acquire();
try
{
Console.WriteLine("TEST: reopen " + s);
}
finally
{
mgr.Release(s);
}
Console.WriteLine("TEST: " + values.Count + " values");
}
}
if (threadRandom.Next(10) == 7)
{
Assert.AreEqual(null, rt.Get("foo"));
}
if (allIDs.Count > 0)
{
string randomID = allIDs[threadRandom.Next(allIDs.Count)];
int? expected = values[randomID];
if (expected == missing)
{
expected = null;
}
Assert.AreEqual(expected, rt.Get(randomID), "id=" + randomID);
}
}
}
catch (Exception t) when(t.IsThrowable())
{
throw RuntimeException.Create(t);
}
}
public override void Run()
{
try
{
IndexReader lastReader = null;
IndexSearcher lastSearcher = null;
while (Operations.DecrementAndGet() >= 0)
{
// bias toward a recently changed doc
int id = rand.Next(100) < 25 ? OuterInstance.LastId : rand.Next(Ndocs);
// when indexing, we update the index, then the model
// so when querying, we should first check the model, and then the index
long val;
DirectoryReader r;
lock (OuterInstance)
{
val = OuterInstance.CommittedModel[id];
r = OuterInstance.Reader;
r.IncRef();
}
if (VERBOSE)
{
Console.WriteLine("TEST: " + Thread.CurrentThread.Name + ": s id=" + id + " val=" + val + " r=" + r.Version);
}
// sreq = req("wt","json", "q","id:"+Integer.toString(id), "omitHeader","true");
IndexSearcher searcher;
if (r == lastReader)
{
// Just re-use lastSearcher, else
// newSearcher may create too many thread
// pools (ExecutorService):
searcher = lastSearcher;
}
else
{
searcher = NewSearcher(
#if FEATURE_INSTANCE_TESTDATA_INITIALIZATION
OuterInstance,
#endif
r);
lastReader = r;
lastSearcher = searcher;
}
Query q = new TermQuery(new Term("id", Convert.ToString(id)));
TopDocs results = searcher.Search(q, 10);
if (results.TotalHits == 0 && Tombstones)
{
// if we couldn't find the doc, look for its tombstone
q = new TermQuery(new Term("id", "-" + Convert.ToString(id)));
results = searcher.Search(q, 1);
if (results.TotalHits == 0)
{
if (val == -1L)
{
// expected... no doc was added yet
r.DecRef();
continue;
}
Assert.Fail("No documents or tombstones found for id " + id + ", expected at least " + val + " reader=" + r);
}
}
if (results.TotalHits == 0 && !Tombstones)
{
// nothing to do - we can't tell anything from a deleted doc without tombstones
}
else
{
// we should have found the document, or its tombstone
if (results.TotalHits != 1)
{
Console.WriteLine("FAIL: hits id:" + id + " val=" + val);
foreach (ScoreDoc sd in results.ScoreDocs)
{
Document doc = r.Document(sd.Doc);
Console.WriteLine(" docID=" + sd.Doc + " id:" + doc.Get("id") + " foundVal=" + doc.Get(OuterInstance.Field));
}
Assert.Fail("id=" + id + " reader=" + r + " totalHits=" + results.TotalHits);
}
Document doc_ = searcher.Doc(results.ScoreDocs[0].Doc);
long foundVal = Convert.ToInt64(doc_.Get(OuterInstance.Field));
if (foundVal < Math.Abs(val))
{
Assert.Fail("foundVal=" + foundVal + " val=" + val + " id=" + id + " reader=" + r);
}
}
r.DecRef();
}
}
catch (Exception e)
{
Operations.Value = ((int)-1L);
Console.WriteLine(Thread.CurrentThread.Name + ": FAILED: unexpected exception");
Console.WriteLine(e.StackTrace);
throw new Exception(e.Message, e);
}
}
public virtual void TestPayloadsPos0()
{
Directory dir = NewDirectory();
RandomIndexWriter writer = new RandomIndexWriter(
#if FEATURE_INSTANCE_TESTDATA_INITIALIZATION
this,
#endif
Random, dir, new MockPayloadAnalyzer());
Document doc = new Document();
doc.Add(new TextField("content", new StringReader("a a b c d e a f g h i j a b k k")));
writer.AddDocument(doc);
IndexReader readerFromWriter = writer.GetReader();
AtomicReader r = SlowCompositeReaderWrapper.Wrap(readerFromWriter);
DocsAndPositionsEnum tp = r.GetTermPositionsEnum(new Term("content", "a"));
int count = 0;
Assert.IsTrue(tp.NextDoc() != DocIdSetIterator.NO_MORE_DOCS);
// "a" occurs 4 times
Assert.AreEqual(4, tp.Freq);
Assert.AreEqual(0, tp.NextPosition());
Assert.AreEqual(1, tp.NextPosition());
Assert.AreEqual(3, tp.NextPosition());
Assert.AreEqual(6, tp.NextPosition());
// only one doc has "a"
Assert.AreEqual(DocIdSetIterator.NO_MORE_DOCS, tp.NextDoc());
IndexSearcher @is = NewSearcher(readerFromWriter);
SpanTermQuery stq1 = new SpanTermQuery(new Term("content", "a"));
SpanTermQuery stq2 = new SpanTermQuery(new Term("content", "k"));
SpanQuery[] sqs = new SpanQuery[] { stq1, stq2 };
SpanNearQuery snq = new SpanNearQuery(sqs, 30, false);
count = 0;
bool sawZero = false;
if (Verbose)
{
Console.WriteLine("\ngetPayloadSpans test");
}
Search.Spans.Spans pspans = MultiSpansWrapper.Wrap(@is.TopReaderContext, snq);
while (pspans.Next())
{
if (Verbose)
{
Console.WriteLine("doc " + pspans.Doc + ": span " + pspans.Start + " to " + pspans.End);
}
var payloads = pspans.GetPayload();
sawZero |= pspans.Start == 0;
foreach (var bytes in payloads)
{
count++;
if (Verbose)
{
Console.WriteLine(" payload: " + Encoding.UTF8.GetString(bytes));
}
}
}
Assert.IsTrue(sawZero);
Assert.AreEqual(5, count);
// System.out.println("\ngetSpans test");
Search.Spans.Spans spans = MultiSpansWrapper.Wrap(@is.TopReaderContext, snq);
count = 0;
sawZero = false;
while (spans.Next())
{
count++;
sawZero |= spans.Start == 0;
// System.out.println(spans.Doc() + " - " + spans.Start() + " - " +
// spans.End());
}
Assert.AreEqual(4, count);
Assert.IsTrue(sawZero);
// System.out.println("\nPayloadSpanUtil test");
sawZero = false;
PayloadSpanUtil psu = new PayloadSpanUtil(@is.TopReaderContext);
var pls = psu.GetPayloadsForQuery(snq);
count = pls.Count;
foreach (var bytes in pls)
{
string s = Encoding.UTF8.GetString(bytes);
//System.out.println(s);
sawZero |= s.Equals("pos: 0", StringComparison.Ordinal);
}
Assert.AreEqual(5, count);
Assert.IsTrue(sawZero);
writer.Dispose();
@is.IndexReader.Dispose();
dir.Dispose();
}
public virtual void Test()
{
// update variables
int commitPercent = Random.Next(20);
int softCommitPercent = Random.Next(100); // what percent of the commits are soft
int deletePercent = Random.Next(50);
int deleteByQueryPercent = Random.Next(25);
int ndocs = AtLeast(50);
int nWriteThreads = TestUtil.NextInt32(Random, 1, TEST_NIGHTLY ? 10 : 5);
int maxConcurrentCommits = TestUtil.NextInt32(Random, 1, TEST_NIGHTLY ? 10 : 5); // number of committers at a time... needed if we want to avoid commit errors due to exceeding the max
bool tombstones = Random.NextBoolean();
// query variables
AtomicInt64 operations = new AtomicInt64(AtLeast(10000)); // number of query operations to perform in total
int nReadThreads = TestUtil.NextInt32(Random, 1, TEST_NIGHTLY ? 10 : 5);
InitModel(ndocs);
FieldType storedOnlyType = new FieldType();
storedOnlyType.IsStored = true;
if (VERBOSE)
{
Console.WriteLine("\n");
Console.WriteLine("TEST: commitPercent=" + commitPercent);
Console.WriteLine("TEST: softCommitPercent=" + softCommitPercent);
Console.WriteLine("TEST: deletePercent=" + deletePercent);
Console.WriteLine("TEST: deleteByQueryPercent=" + deleteByQueryPercent);
Console.WriteLine("TEST: ndocs=" + ndocs);
Console.WriteLine("TEST: nWriteThreads=" + nWriteThreads);
Console.WriteLine("TEST: nReadThreads=" + nReadThreads);
Console.WriteLine("TEST: maxConcurrentCommits=" + maxConcurrentCommits);
Console.WriteLine("TEST: tombstones=" + tombstones);
Console.WriteLine("TEST: operations=" + operations);
Console.WriteLine("\n");
}
AtomicInt32 numCommitting = new AtomicInt32();
IList <ThreadJob> threads = new List <ThreadJob>();
Directory dir = NewDirectory();
RandomIndexWriter writer = new RandomIndexWriter(Random, dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random)));
writer.DoRandomForceMergeAssert = false;
writer.Commit();
Reader = DirectoryReader.Open(dir);
for (int i = 0; i < nWriteThreads; i++)
{
ThreadJob thread = new ThreadAnonymousInnerClassHelper(this, "WRITER" + i, commitPercent, softCommitPercent, deletePercent, deleteByQueryPercent, ndocs, maxConcurrentCommits, tombstones, operations, storedOnlyType, numCommitting, writer);
threads.Add(thread);
}
for (int i = 0; i < nReadThreads; i++)
{
ThreadJob thread = new ThreadAnonymousInnerClassHelper2(this, "READER" + i, ndocs, tombstones, operations);
threads.Add(thread);
}
foreach (ThreadJob thread in threads)
{
thread.Start();
}
foreach (ThreadJob thread in threads)
{
thread.Join();
}
writer.Dispose();
if (VERBOSE)
{
Console.WriteLine("TEST: close reader=" + Reader);
}
Reader.Dispose();
dir.Dispose();
}
public virtual void TestCommitOnCloseForceMerge()
{
Directory dir = NewDirectory();
// Must disable throwing exc on double-write: this
// test uses IW.rollback which easily results in
// writing to same file more than once
if (dir is MockDirectoryWrapper)
{
((MockDirectoryWrapper)dir).PreventDoubleWrite = false;
}
IndexWriter writer = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random)).SetMaxBufferedDocs(10).SetMergePolicy(NewLogMergePolicy(10)));
for (int j = 0; j < 17; j++)
{
AddDocWithIndex(writer, j);
}
writer.Dispose();
writer = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random)).SetOpenMode(OpenMode.APPEND));
writer.ForceMerge(1);
// Open a reader before closing (commiting) the writer:
DirectoryReader reader = DirectoryReader.Open(dir);
// Reader should see index as multi-seg at this
// point:
Assert.IsTrue(reader.Leaves.Count > 1, "Reader incorrectly sees one segment");
reader.Dispose();
// Abort the writer:
writer.Rollback();
TestIndexWriter.AssertNoUnreferencedFiles(dir, "aborted writer after forceMerge");
// Open a reader after aborting writer:
reader = DirectoryReader.Open(dir);
// Reader should still see index as multi-segment
Assert.IsTrue(reader.Leaves.Count > 1, "Reader incorrectly sees one segment");
reader.Dispose();
if (VERBOSE)
{
Console.WriteLine("TEST: do real full merge");
}
writer = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random)).SetOpenMode(OpenMode.APPEND));
writer.ForceMerge(1);
writer.Dispose();
if (VERBOSE)
{
Console.WriteLine("TEST: writer closed");
}
TestIndexWriter.AssertNoUnreferencedFiles(dir, "aborted writer after forceMerge");
// Open a reader after aborting writer:
reader = DirectoryReader.Open(dir);
// Reader should see index as one segment
Assert.AreEqual(1, reader.Leaves.Count, "Reader incorrectly sees more than one segment");
reader.Dispose();
dir.Dispose();
}
private PreviousSearchState AssertSame(IndexSearcher mockSearcher, NodeState.ShardIndexSearcher shardSearcher, Query q, Sort sort, PreviousSearchState state)
{
int numHits = TestUtil.NextInt32(Random, 1, 100);
if (state != null && state.SearchAfterLocal == null)
{
// In addition to what we last searched:
numHits += state.NumHitsPaged;
}
if (VERBOSE)
{
Console.WriteLine("TEST: query=" + q + " sort=" + sort + " numHits=" + numHits);
if (state != null)
{
Console.WriteLine(" prev: searchAfterLocal=" + state.SearchAfterLocal + " searchAfterShard=" + state.SearchAfterShard + " numHitsPaged=" + state.NumHitsPaged);
}
}
// Single (mock local) searcher:
TopDocs hits;
if (sort == null)
{
if (state != null && state.SearchAfterLocal != null)
{
hits = mockSearcher.SearchAfter(state.SearchAfterLocal, q, numHits);
}
else
{
hits = mockSearcher.Search(q, numHits);
}
}
else
{
hits = mockSearcher.Search(q, numHits, sort);
}
// Shard searcher
TopDocs shardHits;
if (sort == null)
{
if (state != null && state.SearchAfterShard != null)
{
shardHits = shardSearcher.SearchAfter(state.SearchAfterShard, q, numHits);
}
else
{
shardHits = shardSearcher.Search(q, numHits);
}
}
else
{
shardHits = shardSearcher.Search(q, numHits, sort);
}
int numNodes = shardSearcher.GetNodeVersions().Length;
int[] @base = new int[numNodes];
IList <IndexReaderContext> subs = mockSearcher.TopReaderContext.Children;
Assert.AreEqual(numNodes, subs.Count);
for (int nodeID = 0; nodeID < numNodes; nodeID++)
{
@base[nodeID] = subs[nodeID].DocBaseInParent;
}
if (VERBOSE)
{
/*
* for(int shardID=0;shardID<shardSearchers.Length;shardID++) {
* System.out.println(" shard=" + shardID + " maxDoc=" + shardSearchers[shardID].searcher.getIndexReader().MaxDoc);
* }
*/
Console.WriteLine(" single searcher: " + hits.TotalHits + " totalHits maxScore=" + hits.MaxScore);
for (int i = 0; i < hits.ScoreDocs.Length; i++)
{
ScoreDoc sd = hits.ScoreDocs[i];
Console.WriteLine(" doc=" + sd.Doc + " score=" + sd.Score);
}
Console.WriteLine(" shard searcher: " + shardHits.TotalHits + " totalHits maxScore=" + shardHits.MaxScore);
for (int i = 0; i < shardHits.ScoreDocs.Length; i++)
{
ScoreDoc sd = shardHits.ScoreDocs[i];
Console.WriteLine(" doc=" + sd.Doc + " (rebased: " + (sd.Doc + @base[sd.ShardIndex]) + ") score=" + sd.Score + " shard=" + sd.ShardIndex);
}
}
int numHitsPaged;
if (state != null && state.SearchAfterLocal != null)
{
numHitsPaged = hits.ScoreDocs.Length;
if (state != null)
{
numHitsPaged += state.NumHitsPaged;
}
}
else
{
numHitsPaged = hits.ScoreDocs.Length;
}
bool moreHits;
ScoreDoc bottomHit;
ScoreDoc bottomHitShards;
if (numHitsPaged < hits.TotalHits)
{
// More hits to page through
moreHits = true;
if (sort == null)
{
bottomHit = hits.ScoreDocs[hits.ScoreDocs.Length - 1];
ScoreDoc sd = shardHits.ScoreDocs[shardHits.ScoreDocs.Length - 1];
// Must copy because below we rebase:
bottomHitShards = new ScoreDoc(sd.Doc, sd.Score, sd.ShardIndex);
if (VERBOSE)
{
Console.WriteLine(" save bottomHit=" + bottomHit);
}
}
else
{
bottomHit = null;
bottomHitShards = null;
}
}
else
{
Assert.AreEqual(hits.TotalHits, numHitsPaged);
bottomHit = null;
bottomHitShards = null;
moreHits = false;
}
// Must rebase so Assert.AreEqual passes:
for (int hitID = 0; hitID < shardHits.ScoreDocs.Length; hitID++)
{
ScoreDoc sd = shardHits.ScoreDocs[hitID];
sd.Doc += @base[sd.ShardIndex];
}
TestUtil.AssertEquals(hits, shardHits);
if (moreHits)
{
// Return a continuation:
return(new PreviousSearchState(q, sort, bottomHit, bottomHitShards, shardSearcher.GetNodeVersions(), numHitsPaged));
}
else
{
return(null);
}
}
