public virtual void TestSimple() { int numNodes = TestUtil.NextInt32(Random, 1, 10); double runTimeSec = AtLeast(3); int minDocsToMakeTerms = TestUtil.NextInt32(Random, 5, 20); int maxSearcherAgeSeconds = TestUtil.NextInt32(Random, 1, 3); if (VERBOSE) { Console.WriteLine("TEST: numNodes=" + numNodes + " runTimeSec=" + runTimeSec + " maxSearcherAgeSeconds=" + maxSearcherAgeSeconds); } Start(numNodes, runTimeSec, maxSearcherAgeSeconds); List <PreviousSearchState> priorSearches = new List <PreviousSearchState>(); List <BytesRef> terms = null; while (Time.NanoTime() < endTimeNanos) { bool doFollowon = priorSearches.Count > 0 && Random.Next(7) == 1; // Pick a random node; we will run the query on this node: int myNodeID = Random.Next(numNodes); NodeState.ShardIndexSearcher localShardSearcher; PreviousSearchState prevSearchState; if (doFollowon) { // Pretend user issued a followon query: prevSearchState = priorSearches[Random.Next(priorSearches.Count)]; if (VERBOSE) { Console.WriteLine("\nTEST: follow-on query age=" + ((Time.NanoTime() - prevSearchState.SearchTimeNanos) / 1000000000.0)); } try { localShardSearcher = m_nodes[myNodeID].Acquire(prevSearchState.Versions); } catch (SearcherExpiredException see) { // Expected, sometimes; in a "real" app we would // either forward this error to the user ("too // much time has passed; please re-run your // search") or sneakily just switch to newest // searcher w/o telling them... if (VERBOSE) { Console.WriteLine(" searcher expired during local shard searcher init: " + see); } priorSearches.Remove(prevSearchState); continue; } } else { if (VERBOSE) { Console.WriteLine("\nTEST: fresh query"); } // Do fresh query: localShardSearcher = m_nodes[myNodeID].Acquire(); prevSearchState = null; } IndexReader[] subs = new IndexReader[numNodes]; PreviousSearchState searchState = null; try { // Mock: now make a single reader (MultiReader) from all node // searchers. In a real shard env you can't do this... we // do it to confirm results from the shard searcher // are correct: int docCount = 0; try { for (int nodeID = 0; nodeID < numNodes; nodeID++) { long subVersion = localShardSearcher.GetNodeVersions()[nodeID]; IndexSearcher sub = m_nodes[nodeID].Searchers.Acquire(subVersion); if (sub == null) { nodeID--; while (nodeID >= 0) { subs[nodeID].DecRef(); subs[nodeID] = null; nodeID--; } throw new SearcherExpiredException("nodeID=" + nodeID + " version=" + subVersion); } subs[nodeID] = sub.IndexReader; docCount += subs[nodeID].MaxDoc; } } catch (SearcherExpiredException see) { // Expected if (VERBOSE) { Console.WriteLine(" searcher expired during mock reader init: " + see); } continue; } IndexReader mockReader = new MultiReader(subs); IndexSearcher mockSearcher = new IndexSearcher(mockReader); Query query; Sort sort; if (prevSearchState != null) { query = prevSearchState.Query; sort = prevSearchState.Sort; } else { if (terms == null && docCount > minDocsToMakeTerms) { // TODO: try to "focus" on high freq terms sometimes too // TODO: maybe also periodically reset the terms...? TermsEnum termsEnum = MultiFields.GetTerms(mockReader, "body").GetIterator(null); terms = new List <BytesRef>(); while (termsEnum.Next() != null) { terms.Add(BytesRef.DeepCopyOf(termsEnum.Term)); } if (VERBOSE) { Console.WriteLine("TEST: init terms: " + terms.Count + " terms"); } if (terms.Count == 0) { terms = null; } } if (VERBOSE) { Console.WriteLine(" maxDoc=" + mockReader.MaxDoc); } if (terms != null) { if (Random.NextBoolean()) { query = new TermQuery(new Term("body", terms[Random.Next(terms.Count)])); } else { string t = terms[Random.Next(terms.Count)].Utf8ToString(); string prefix; if (t.Length <= 1) { prefix = t; } else { prefix = t.Substring(0, TestUtil.NextInt32(Random, 1, 2)); } query = new PrefixQuery(new Term("body", prefix)); } if (Random.NextBoolean()) { sort = null; } else { // TODO: sort by more than 1 field int what = Random.Next(3); if (what == 0) { sort = new Sort(SortField.FIELD_SCORE); } else if (what == 1) { // TODO: this sort doesn't merge // correctly... it's tricky because you // could have > 2.1B docs across all shards: //sort = new Sort(SortField.FIELD_DOC); sort = null; } else if (what == 2) { sort = new Sort(new SortField[] { new SortField("docid", SortFieldType.INT32, Random.NextBoolean()) }); } else { sort = new Sort(new SortField[] { new SortField("title", SortFieldType.STRING, Random.NextBoolean()) }); } } } else { query = null; sort = null; } } if (query != null) { try { searchState = AssertSame(mockSearcher, localShardSearcher, query, sort, prevSearchState); } catch (SearcherExpiredException see) { // Expected; in a "real" app we would // either forward this error to the user ("too // much time has passed; please re-run your // search") or sneakily just switch to newest // searcher w/o telling them... if (VERBOSE) { Console.WriteLine(" searcher expired during search: " + see); Console.Out.Write(see.StackTrace); } // We can't do this in general: on a very slow // computer it's possible the local searcher // expires before we can finish our search: // assert prevSearchState != null; if (prevSearchState != null) { priorSearches.Remove(prevSearchState); } } } } finally { m_nodes[myNodeID].Release(localShardSearcher); foreach (IndexReader sub in subs) { if (sub != null) { sub.DecRef(); } } } if (searchState != null && searchState.SearchAfterLocal != null && Random.Next(5) == 3) { priorSearches.Add(searchState); if (priorSearches.Count > 200) { priorSearches.Shuffle(Random); priorSearches.SubList(100, priorSearches.Count).Clear(); } } } Finish(); }
private void TestRandomTrieAndClassicRangeQuery(int precisionStep) { string field = "field" + precisionStep; int totalTermCountT = 0, totalTermCountC = 0, termCountT, termCountC; int num = TestUtil.NextInt32(Random, 10, 20); for (int i = 0; i < num; i++) { long lower = (long)(Random.NextDouble() * NoDocs * Distance) + StartOffset; long upper = (long)(Random.NextDouble() * NoDocs * Distance) + StartOffset; if (lower > upper) { long a = lower; lower = upper; upper = a; } BytesRef lowerBytes = new BytesRef(NumericUtils.BUF_SIZE_INT64), upperBytes = new BytesRef(NumericUtils.BUF_SIZE_INT64); NumericUtils.Int64ToPrefixCodedBytes(lower, 0, lowerBytes); NumericUtils.Int64ToPrefixCodedBytes(upper, 0, upperBytes); // test inclusive range NumericRangeQuery <long> tq = NumericRangeQuery.NewInt64Range(field, precisionStep, lower, upper, true, true); TermRangeQuery cq = new TermRangeQuery(field, lowerBytes, upperBytes, true, true); TopDocs tTopDocs = Searcher.Search(tq, 1); TopDocs cTopDocs = Searcher.Search(cq, 1); Assert.AreEqual(cTopDocs.TotalHits, tTopDocs.TotalHits, "Returned count for NumericRangeQuery and TermRangeQuery must be equal"); totalTermCountT += termCountT = CountTerms(tq); totalTermCountC += termCountC = CountTerms(cq); CheckTermCounts(precisionStep, termCountT, termCountC); // test exclusive range tq = NumericRangeQuery.NewInt64Range(field, precisionStep, lower, upper, false, false); cq = new TermRangeQuery(field, lowerBytes, upperBytes, false, false); tTopDocs = Searcher.Search(tq, 1); cTopDocs = Searcher.Search(cq, 1); Assert.AreEqual(cTopDocs.TotalHits, tTopDocs.TotalHits, "Returned count for NumericRangeQuery and TermRangeQuery must be equal"); totalTermCountT += termCountT = CountTerms(tq); totalTermCountC += termCountC = CountTerms(cq); CheckTermCounts(precisionStep, termCountT, termCountC); // test left exclusive range tq = NumericRangeQuery.NewInt64Range(field, precisionStep, lower, upper, false, true); cq = new TermRangeQuery(field, lowerBytes, upperBytes, false, true); tTopDocs = Searcher.Search(tq, 1); cTopDocs = Searcher.Search(cq, 1); Assert.AreEqual(cTopDocs.TotalHits, tTopDocs.TotalHits, "Returned count for NumericRangeQuery and TermRangeQuery must be equal"); totalTermCountT += termCountT = CountTerms(tq); totalTermCountC += termCountC = CountTerms(cq); CheckTermCounts(precisionStep, termCountT, termCountC); // test right exclusive range tq = NumericRangeQuery.NewInt64Range(field, precisionStep, lower, upper, true, false); cq = new TermRangeQuery(field, lowerBytes, upperBytes, true, false); tTopDocs = Searcher.Search(tq, 1); cTopDocs = Searcher.Search(cq, 1); Assert.AreEqual(cTopDocs.TotalHits, tTopDocs.TotalHits, "Returned count for NumericRangeQuery and TermRangeQuery must be equal"); totalTermCountT += termCountT = CountTerms(tq); totalTermCountC += termCountC = CountTerms(cq); CheckTermCounts(precisionStep, termCountT, termCountC); } CheckTermCounts(precisionStep, totalTermCountT, totalTermCountC); if (VERBOSE && precisionStep != int.MaxValue) { Console.WriteLine("Average number of terms during random search on '" + field + "':"); Console.WriteLine(" Numeric query: " + (((double)totalTermCountT) / (num * 4))); Console.WriteLine(" Classical query: " + (((double)totalTermCountC) / (num * 4))); } }
public virtual void TestRandom() { AssumeTrue("Test requires SortedSetDV support", DefaultCodecSupportsSortedSet); string[] tokens = GetRandomTokens(10); Directory indexDir = NewDirectory(); Directory taxoDir = NewDirectory(); RandomIndexWriter w = new RandomIndexWriter( #if FEATURE_INSTANCE_TESTDATA_INITIALIZATION this, #endif Random, indexDir); FacetsConfig config = new FacetsConfig(); int numDocs = AtLeast(1000); int numDims = TestUtil.NextInt32(Random, 1, 7); IList <TestDoc> testDocs = GetRandomDocs(tokens, numDocs, numDims); foreach (TestDoc testDoc in testDocs) { Document doc = new Document(); doc.Add(NewStringField("content", testDoc.content, Field.Store.NO)); for (int j = 0; j < numDims; j++) { if (testDoc.dims[j] != null) { doc.Add(new SortedSetDocValuesFacetField("dim" + j, testDoc.dims[j])); } } w.AddDocument(config.Build(doc)); } // NRT open IndexSearcher searcher = NewSearcher(w.GetReader()); // Per-top-reader state: SortedSetDocValuesReaderState state = new DefaultSortedSetDocValuesReaderState(searcher.IndexReader); int iters = AtLeast(100); for (int iter = 0; iter < iters; iter++) { string searchToken = tokens[Random.Next(tokens.Length)]; if (VERBOSE) { Console.WriteLine("\nTEST: iter content=" + searchToken); } FacetsCollector fc = new FacetsCollector(); FacetsCollector.Search(searcher, new TermQuery(new Term("content", searchToken)), 10, fc); Facets facets = new SortedSetDocValuesFacetCounts(state, fc); // Slow, yet hopefully bug-free, faceting: var expectedCounts = new List <Dictionary <string, int?> >(); for (int i = 0; i < numDims; i++) { expectedCounts.Add(new Dictionary <string, int?>()); } foreach (TestDoc doc in testDocs) { if (doc.content.Equals(searchToken, StringComparison.Ordinal)) { for (int j = 0; j < numDims; j++) { if (doc.dims[j] != null) { int?v; if (!expectedCounts[j].TryGetValue(doc.dims[j], out v)) { expectedCounts[j][doc.dims[j]] = 1; } else { expectedCounts[j][doc.dims[j]] = (int)v + 1; } } } } } List <FacetResult> expected = new List <FacetResult>(); for (int i = 0; i < numDims; i++) { List <LabelAndValue> labelValues = new List <LabelAndValue>(); int totCount = 0; foreach (KeyValuePair <string, int?> ent in expectedCounts[i]) { labelValues.Add(new LabelAndValue(ent.Key, ent.Value.Value)); totCount += ent.Value.Value; } SortLabelValues(labelValues); if (totCount > 0) { expected.Add(new FacetResult("dim" + i, new string[0], totCount, labelValues.ToArray(), labelValues.Count)); } } // Sort by highest value, tie break by value: SortFacetResults(expected); IList <FacetResult> actual = facets.GetAllDims(10); // Messy: fixup ties //sortTies(actual); CollectionAssert.AreEqual(expected, actual); } IOUtils.Dispose(w, searcher.IndexReader, indexDir, taxoDir); }
public virtual void TestRandom() { int iters = AtLeast(10); for (int iter = 0; iter < iters; iter++) { int numBytes = TestUtil.NextInt32(Random, 1, 200000); byte[] expected = new byte[numBytes]; int blockBits = TestUtil.NextInt32(Random, 8, 15); BytesStore bytes = new BytesStore(blockBits); if (VERBOSE) { Console.WriteLine("TEST: iter=" + iter + " numBytes=" + numBytes + " blockBits=" + blockBits); } int pos = 0; while (pos < numBytes) { int op = Random.Next(8); if (VERBOSE) { Console.WriteLine(" cycle pos=" + pos); } switch (op) { case 0: { // write random byte byte b = (byte)Random.Next(256); if (VERBOSE) { Console.WriteLine(" writeByte b=" + b); } expected[pos++] = b; bytes.WriteByte(b); } break; case 1: { // write random byte[] int len = Random.Next(Math.Min(numBytes - pos, 100)); byte[] temp = new byte[len]; Random.NextBytes(temp); if (VERBOSE) { Console.WriteLine(" writeBytes len=" + len + " bytes=" + Arrays.ToString(temp)); } Array.Copy(temp, 0, expected, pos, temp.Length); bytes.WriteBytes(temp, 0, temp.Length); pos += len; } break; case 2: { // write int @ absolute pos if (pos > 4) { int x = Random.Next(); int randomPos = Random.Next(pos - 4); if (VERBOSE) { Console.WriteLine(" abs writeInt pos=" + randomPos + " x=" + x); } bytes.WriteInt32(randomPos, x); expected[randomPos++] = (byte)(x >> 24); expected[randomPos++] = (byte)(x >> 16); expected[randomPos++] = (byte)(x >> 8); expected[randomPos++] = (byte)x; } } break; case 3: { // reverse bytes if (pos > 1) { int len = TestUtil.NextInt32(Random, 2, Math.Min(100, pos)); int start; if (len == pos) { start = 0; } else { start = Random.Next(pos - len); } int end = start + len - 1; if (VERBOSE) { Console.WriteLine(" reverse start=" + start + " end=" + end + " len=" + len + " pos=" + pos); } bytes.Reverse(start, end); while (start <= end) { byte b = expected[end]; expected[end] = expected[start]; expected[start] = b; start++; end--; } } } break; case 4: { // abs write random byte[] if (pos > 2) { int randomPos = Random.Next(pos - 1); int len = TestUtil.NextInt32(Random, 1, Math.Min(pos - randomPos - 1, 100)); byte[] temp = new byte[len]; Random.NextBytes(temp); if (VERBOSE) { Console.WriteLine(" abs writeBytes pos=" + randomPos + " len=" + len + " bytes=" + Arrays.ToString(temp)); } Array.Copy(temp, 0, expected, randomPos, temp.Length); bytes.WriteBytes(randomPos, temp, 0, temp.Length); } } break; case 5: { // copyBytes if (pos > 1) { int src = Random.Next(pos - 1); int dest = TestUtil.NextInt32(Random, src + 1, pos - 1); int len = TestUtil.NextInt32(Random, 1, Math.Min(300, pos - dest)); if (VERBOSE) { Console.WriteLine(" copyBytes src=" + src + " dest=" + dest + " len=" + len); } Array.Copy(expected, src, expected, dest, len); bytes.CopyBytes(src, dest, len); } } break; case 6: { // skip int len = Random.Next(Math.Min(100, numBytes - pos)); if (VERBOSE) { Console.WriteLine(" skip len=" + len); } pos += len; bytes.SkipBytes(len); // NOTE: must fill in zeros in case truncate was // used, else we get false fails: if (len > 0) { byte[] zeros = new byte[len]; bytes.WriteBytes(pos - len, zeros, 0, len); } } break; case 7: { // absWriteByte if (pos > 0) { int dest = Random.Next(pos); byte b = (byte)Random.Next(256); expected[dest] = b; bytes.WriteByte(dest, b); } break; } } Assert.AreEqual(pos, bytes.Position); if (pos > 0 && Random.Next(50) == 17) { // truncate int len = TestUtil.NextInt32(Random, 1, Math.Min(pos, 100)); bytes.Truncate(pos - len); pos -= len; Arrays.Fill(expected, pos, pos + len, (byte)0); if (VERBOSE) { Console.WriteLine(" truncate len=" + len + " newPos=" + pos); } } if ((pos > 0 && Random.Next(200) == 17)) { Verify(bytes, expected, pos); } } BytesStore bytesToVerify; if (Random.NextBoolean()) { if (VERBOSE) { Console.WriteLine("TEST: save/load final bytes"); } Directory dir = NewDirectory(); IndexOutput @out = dir.CreateOutput("bytes", IOContext.DEFAULT); bytes.WriteTo(@out); @out.Dispose(); IndexInput @in = dir.OpenInput("bytes", IOContext.DEFAULT); bytesToVerify = new BytesStore(@in, numBytes, TestUtil.NextInt32(Random, 256, int.MaxValue)); @in.Dispose(); dir.Dispose(); } else { bytesToVerify = bytes; } Verify(bytesToVerify, expected, numBytes); } }
public virtual void TestArbitraryFields() { Directory dir = NewDirectory(); RandomIndexWriter w = new RandomIndexWriter( #if FEATURE_INSTANCE_TESTDATA_INITIALIZATION this, #endif Random, dir); int NUM_DOCS = AtLeast(27); if (Verbose) { Console.WriteLine("TEST: " + NUM_DOCS + " docs"); } int[] fieldsPerDoc = new int[NUM_DOCS]; int baseCount = 0; for (int docCount = 0; docCount < NUM_DOCS; docCount++) { int fieldCount = TestUtil.NextInt32(Random, 1, 17); fieldsPerDoc[docCount] = fieldCount - 1; int finalDocCount = docCount; if (Verbose) { Console.WriteLine("TEST: " + fieldCount + " fields in doc " + docCount); } int finalBaseCount = baseCount; baseCount += fieldCount - 1; w.AddDocument(new IterableAnonymousInnerClassHelper(this, fieldCount, finalDocCount, finalBaseCount)); } IndexReader r = w.GetReader(); w.Dispose(); IndexSearcher s = NewSearcher(r); int counter = 0; for (int id = 0; id < NUM_DOCS; id++) { if (Verbose) { Console.WriteLine("TEST: verify doc id=" + id + " (" + fieldsPerDoc[id] + " fields) counter=" + counter); } TopDocs hits = s.Search(new TermQuery(new Term("id", "" + id)), 1); Assert.AreEqual(1, hits.TotalHits); int docID = hits.ScoreDocs[0].Doc; Document doc = s.Doc(docID); int endCounter = counter + fieldsPerDoc[id]; while (counter < endCounter) { string name = "f" + counter; int fieldID = counter % 10; bool stored = (counter & 1) == 0 || fieldID == 3; bool binary = fieldID == 3; bool indexed = fieldID != 3; string stringValue; if (fieldID != 3 && fieldID != 9) { stringValue = "text " + counter; } else { stringValue = null; } // stored: if (stored) { IIndexableField f = doc.GetField(name); Assert.IsNotNull(f, "doc " + id + " doesn't have field f" + counter); if (binary) { Assert.IsNotNull(f, "doc " + id + " doesn't have field f" + counter); BytesRef b = f.GetBinaryValue(); Assert.IsNotNull(b); Assert.AreEqual(10, b.Length); for (int idx = 0; idx < 10; idx++) { Assert.AreEqual((byte)(idx + counter), b.Bytes[b.Offset + idx]); } } else { if (Debugging.AssertsEnabled) { Debugging.Assert(stringValue != null); } Assert.AreEqual(stringValue, f.GetStringValue()); } } if (indexed) { bool tv = counter % 2 == 1 && fieldID != 9; if (tv) { Terms tfv = r.GetTermVectors(docID).GetTerms(name); Assert.IsNotNull(tfv); TermsEnum termsEnum = tfv.GetIterator(null); Assert.AreEqual(new BytesRef("" + counter), termsEnum.Next()); Assert.AreEqual(1, termsEnum.TotalTermFreq); DocsAndPositionsEnum dpEnum = termsEnum.DocsAndPositions(null, null); Assert.IsTrue(dpEnum.NextDoc() != DocIdSetIterator.NO_MORE_DOCS); Assert.AreEqual(1, dpEnum.Freq); Assert.AreEqual(1, dpEnum.NextPosition()); Assert.AreEqual(new BytesRef("text"), termsEnum.Next()); Assert.AreEqual(1, termsEnum.TotalTermFreq); dpEnum = termsEnum.DocsAndPositions(null, dpEnum); Assert.IsTrue(dpEnum.NextDoc() != DocIdSetIterator.NO_MORE_DOCS); Assert.AreEqual(1, dpEnum.Freq); Assert.AreEqual(0, dpEnum.NextPosition()); Assert.IsNull(termsEnum.Next()); // TODO: offsets } else { Fields vectors = r.GetTermVectors(docID); Assert.IsTrue(vectors == null || vectors.GetTerms(name) == null); } BooleanQuery bq = new BooleanQuery(); bq.Add(new TermQuery(new Term("id", "" + id)), Occur.MUST); bq.Add(new TermQuery(new Term(name, "text")), Occur.MUST); TopDocs hits2 = s.Search(bq, 1); Assert.AreEqual(1, hits2.TotalHits); Assert.AreEqual(docID, hits2.ScoreDocs[0].Doc); bq = new BooleanQuery(); bq.Add(new TermQuery(new Term("id", "" + id)), Occur.MUST); bq.Add(new TermQuery(new Term(name, "" + counter)), Occur.MUST); TopDocs hits3 = s.Search(bq, 1); Assert.AreEqual(1, hits3.TotalHits); Assert.AreEqual(docID, hits3.ScoreDocs[0].Doc); } counter++; } } r.Dispose(); dir.Dispose(); }
public virtual void TestIntersectRandom() { Directory dir = NewDirectory(); RandomIndexWriter w = new RandomIndexWriter( #if FEATURE_INSTANCE_TESTDATA_INITIALIZATION this, #endif Random, dir); int numTerms = AtLeast(300); //final int numTerms = 50; ISet <string> terms = new JCG.HashSet <string>(); ICollection <string> pendingTerms = new List <string>(); IDictionary <BytesRef, int?> termToID = new Dictionary <BytesRef, int?>(); int id = 0; while (terms.Count != numTerms) { string s = RandomString; if (!terms.Contains(s)) { terms.Add(s); pendingTerms.Add(s); if (Random.Next(20) == 7) { AddDoc(w, pendingTerms, termToID, id++); } } } AddDoc(w, pendingTerms, termToID, id++); BytesRef[] termsArray = new BytesRef[terms.Count]; ISet <BytesRef> termsSet = new JCG.HashSet <BytesRef>(); { int upto = 0; foreach (string s in terms) { BytesRef b = new BytesRef(s); termsArray[upto++] = b; termsSet.Add(b); } Array.Sort(termsArray); } if (VERBOSE) { Console.WriteLine("\nTEST: indexed terms (unicode order):"); foreach (BytesRef t in termsArray) { Console.WriteLine(" " + t.Utf8ToString() + " -> id:" + termToID[t]); } } IndexReader r = w.GetReader(); w.Dispose(); // NOTE: intentional insanity!! FieldCache.Int32s docIDToID = FieldCache.DEFAULT.GetInt32s(SlowCompositeReaderWrapper.Wrap(r), "id", false); for (int iter = 0; iter < 10 * RANDOM_MULTIPLIER; iter++) { // TODO: can we also test infinite As here...? // From the random terms, pick some ratio and compile an // automaton: ISet <string> acceptTerms = new JCG.HashSet <string>(); JCG.SortedSet <BytesRef> sortedAcceptTerms = new JCG.SortedSet <BytesRef>(); double keepPct = Random.NextDouble(); Automaton a; if (iter == 0) { if (VERBOSE) { Console.WriteLine("\nTEST: empty automaton"); } a = BasicAutomata.MakeEmpty(); } else { if (VERBOSE) { Console.WriteLine("\nTEST: keepPct=" + keepPct); } foreach (string s in terms) { string s2; if (Random.NextDouble() <= keepPct) { s2 = s; } else { s2 = RandomString; } acceptTerms.Add(s2); sortedAcceptTerms.Add(new BytesRef(s2)); } a = BasicAutomata.MakeStringUnion(sortedAcceptTerms); } if (Random.NextBoolean()) { if (VERBOSE) { Console.WriteLine("TEST: reduce the automaton"); } a.Reduce(); } CompiledAutomaton c = new CompiledAutomaton(a, true, false); BytesRef[] acceptTermsArray = new BytesRef[acceptTerms.Count]; ISet <BytesRef> acceptTermsSet = new JCG.HashSet <BytesRef>(); int upto = 0; foreach (string s in acceptTerms) { BytesRef b = new BytesRef(s); acceptTermsArray[upto++] = b; acceptTermsSet.Add(b); Assert.IsTrue(Accepts(c, b)); } Array.Sort(acceptTermsArray); if (VERBOSE) { Console.WriteLine("\nTEST: accept terms (unicode order):"); foreach (BytesRef t in acceptTermsArray) { Console.WriteLine(" " + t.Utf8ToString() + (termsSet.Contains(t) ? " (exists)" : "")); } Console.WriteLine(a.ToDot()); } for (int iter2 = 0; iter2 < 100; iter2++) { BytesRef startTerm = acceptTermsArray.Length == 0 || Random.NextBoolean() ? null : acceptTermsArray[Random.Next(acceptTermsArray.Length)]; if (VERBOSE) { Console.WriteLine("\nTEST: iter2=" + iter2 + " startTerm=" + (startTerm == null ? "<null>" : startTerm.Utf8ToString())); if (startTerm != null) { int state = c.RunAutomaton.InitialState; for (int idx = 0; idx < startTerm.Length; idx++) { int label = startTerm.Bytes[startTerm.Offset + idx] & 0xff; Console.WriteLine(" state=" + state + " label=" + label); state = c.RunAutomaton.Step(state, label); Assert.IsTrue(state != -1); } Console.WriteLine(" state=" + state); } } TermsEnum te = MultiFields.GetTerms(r, "f").Intersect(c, startTerm); int loc; if (startTerm == null) { loc = 0; } else { loc = Array.BinarySearch(termsArray, BytesRef.DeepCopyOf(startTerm)); if (loc < 0) { loc = -(loc + 1); } else { // startTerm exists in index loc++; } } while (loc < termsArray.Length && !acceptTermsSet.Contains(termsArray[loc])) { loc++; } DocsEnum docsEnum = null; while (loc < termsArray.Length) { BytesRef expected = termsArray[loc]; BytesRef actual = te.Next(); if (VERBOSE) { Console.WriteLine("TEST: next() expected=" + expected.Utf8ToString() + " actual=" + (actual == null ? "null" : actual.Utf8ToString())); } Assert.AreEqual(expected, actual); Assert.AreEqual(1, te.DocFreq); docsEnum = TestUtil.Docs(Random, te, null, docsEnum, DocsFlags.NONE); int docID = docsEnum.NextDoc(); Assert.IsTrue(docID != DocIdSetIterator.NO_MORE_DOCS); Assert.AreEqual(docIDToID.Get(docID), (int)termToID[expected]); do { loc++; } while (loc < termsArray.Length && !acceptTermsSet.Contains(termsArray[loc])); } Assert.IsNull(te.Next()); } } r.Dispose(); dir.Dispose(); }
private void TestRandomSeeks(IndexReader r, params string[] validTermStrings) { BytesRef[] validTerms = new BytesRef[validTermStrings.Length]; for (int termIDX = 0; termIDX < validTermStrings.Length; termIDX++) { validTerms[termIDX] = new BytesRef(validTermStrings[termIDX]); } Array.Sort(validTerms); if (VERBOSE) { Console.WriteLine("TEST: " + validTerms.Length + " terms:"); foreach (BytesRef t in validTerms) { Console.WriteLine(" " + t.Utf8ToString() + " " + t); } } TermsEnum te = MultiFields.GetTerms(r, FIELD).GetIterator(null); int END_LOC = -validTerms.Length - 1; IList <TermAndState> termStates = new List <TermAndState>(); for (int iter = 0; iter < 100 * RANDOM_MULTIPLIER; iter++) { BytesRef t; int loc; TermState termState; if (Random.Next(6) == 4) { // pick term that doens't exist: t = GetNonExistTerm(validTerms); termState = null; if (VERBOSE) { Console.WriteLine("\nTEST: invalid term=" + t.Utf8ToString()); } loc = Array.BinarySearch(validTerms, t); } else if (termStates.Count != 0 && Random.Next(4) == 1) { TermAndState ts = termStates[Random.Next(termStates.Count)]; t = ts.Term; loc = Array.BinarySearch(validTerms, t); Assert.IsTrue(loc >= 0); termState = ts.State; if (VERBOSE) { Console.WriteLine("\nTEST: valid termState term=" + t.Utf8ToString()); } } else { // pick valid term loc = Random.Next(validTerms.Length); t = BytesRef.DeepCopyOf(validTerms[loc]); termState = null; if (VERBOSE) { Console.WriteLine("\nTEST: valid term=" + t.Utf8ToString()); } } // seekCeil or seekExact: bool doSeekExact = Random.NextBoolean(); if (termState != null) { if (VERBOSE) { Console.WriteLine(" seekExact termState"); } te.SeekExact(t, termState); } else if (doSeekExact) { if (VERBOSE) { Console.WriteLine(" seekExact"); } Assert.AreEqual(loc >= 0, te.SeekExact(t)); } else { if (VERBOSE) { Console.WriteLine(" seekCeil"); } TermsEnum.SeekStatus result = te.SeekCeil(t); if (VERBOSE) { Console.WriteLine(" got " + result); } if (loc >= 0) { Assert.AreEqual(TermsEnum.SeekStatus.FOUND, result); } else if (loc == END_LOC) { Assert.AreEqual(TermsEnum.SeekStatus.END, result); } else { Debug.Assert(loc >= -validTerms.Length); Assert.AreEqual(TermsEnum.SeekStatus.NOT_FOUND, result); } } if (loc >= 0) { Assert.AreEqual(t, te.Term); } else if (doSeekExact) { // TermsEnum is unpositioned if seekExact returns false continue; } else if (loc == END_LOC) { continue; } else { loc = -loc - 1; Assert.AreEqual(validTerms[loc], te.Term); } // Do a bunch of next's after the seek int numNext = Random.Next(validTerms.Length); for (int nextCount = 0; nextCount < numNext; nextCount++) { if (VERBOSE) { Console.WriteLine("\nTEST: next loc=" + loc + " of " + validTerms.Length); } BytesRef t2 = te.Next(); loc++; if (loc == validTerms.Length) { Assert.IsNull(t2); break; } else { Assert.AreEqual(validTerms[loc], t2); if (Random.Next(40) == 17 && termStates.Count < 100) { termStates.Add(new TermAndState(validTerms[loc], te.GetTermState())); } } } } }
public virtual void TestCustomDoublesValueSource() { Directory dir = NewDirectory(); RandomIndexWriter writer = new RandomIndexWriter( #if FEATURE_INSTANCE_TESTDATA_INITIALIZATION this, #endif Random, dir); Document doc = new Document(); writer.AddDocument(doc); writer.AddDocument(doc); writer.AddDocument(doc); // Test wants 3 docs in one segment: writer.ForceMerge(1); var vs = new ValueSourceAnonymousInnerClassHelper(this, doc); FacetsConfig config = new FacetsConfig(); FacetsCollector fc = new FacetsCollector(); IndexReader r = writer.GetReader(); IndexSearcher s = NewSearcher(r); s.Search(new MatchAllDocsQuery(), fc); DoubleRange[] ranges = new DoubleRange[] { new DoubleRange("< 1", 0.0, true, 1.0, false), new DoubleRange("< 2", 0.0, true, 2.0, false), new DoubleRange("< 5", 0.0, true, 5.0, false), new DoubleRange("< 10", 0.0, true, 10.0, false), new DoubleRange("< 20", 0.0, true, 20.0, false), new DoubleRange("< 50", 0.0, true, 50.0, false) }; Filter fastMatchFilter; AtomicBoolean filterWasUsed = new AtomicBoolean(); if (Random.NextBoolean()) { // Sort of silly: fastMatchFilter = new CachingWrapperFilterAnonymousInnerClassHelper(this, new QueryWrapperFilter(new MatchAllDocsQuery()), filterWasUsed); } else { fastMatchFilter = null; } if (VERBOSE) { Console.WriteLine("TEST: fastMatchFilter=" + fastMatchFilter); } Facets facets = new DoubleRangeFacetCounts("field", vs, fc, fastMatchFilter, ranges); Assert.AreEqual("dim=field path=[] value=3 childCount=6\n < 1 (0)\n < 2 (1)\n < 5 (3)\n < 10 (3)\n < 20 (3)\n < 50 (3)\n", facets.GetTopChildren(10, "field").ToString()); Assert.True(fastMatchFilter == null || filterWasUsed); DrillDownQuery ddq = new DrillDownQuery(config); ddq.Add("field", ranges[1].GetFilter(fastMatchFilter, vs)); // Test simple drill-down: Assert.AreEqual(1, s.Search(ddq, 10).TotalHits); // Test drill-sideways after drill-down DrillSideways ds = new DrillSidewaysAnonymousInnerClassHelper2(this, s, config, (TaxonomyReader)null, vs, ranges, fastMatchFilter); DrillSidewaysResult dsr = ds.Search(ddq, 10); Assert.AreEqual(1, dsr.Hits.TotalHits); Assert.AreEqual("dim=field path=[] value=3 childCount=6\n < 1 (0)\n < 2 (1)\n < 5 (3)\n < 10 (3)\n < 20 (3)\n < 50 (3)\n", dsr.Facets.GetTopChildren(10, "field").ToString()); IOUtils.Dispose(r, writer, dir); }
public virtual void TestMixedRangeAndNonRangeTaxonomy() { Directory d = NewDirectory(); RandomIndexWriter w = new RandomIndexWriter( #if FEATURE_INSTANCE_TESTDATA_INITIALIZATION this, #endif Random, d); Directory td = NewDirectory(); DirectoryTaxonomyWriter tw = new DirectoryTaxonomyWriter(td, OpenMode.CREATE); FacetsConfig config = new FacetsConfig(); for (long l = 0; l < 100; l++) { Document doc = new Document(); // For computing range facet counts: doc.Add(new NumericDocValuesField("field", l)); // For drill down by numeric range: doc.Add(new Int64Field("field", l, Field.Store.NO)); if ((l & 3) == 0) { doc.Add(new FacetField("dim", "a")); } else { doc.Add(new FacetField("dim", "b")); } w.AddDocument(config.Build(tw, doc)); } IndexReader r = w.GetReader(); var tr = new DirectoryTaxonomyReader(tw); IndexSearcher s = NewSearcher(r); if (VERBOSE) { Console.WriteLine("TEST: searcher=" + s); } DrillSideways ds = new DrillSidewaysAnonymousInnerClassHelper(this, s, config, tr); // First search, no drill downs: DrillDownQuery ddq = new DrillDownQuery(config); DrillSidewaysResult dsr = ds.Search(null, ddq, 10); Assert.AreEqual(100, dsr.Hits.TotalHits); Assert.AreEqual("dim=dim path=[] value=100 childCount=2\n b (75)\n a (25)\n", dsr.Facets.GetTopChildren(10, "dim").ToString()); Assert.AreEqual("dim=field path=[] value=21 childCount=5\n less than 10 (10)\n less than or equal to 10 (11)\n over 90 (9)\n 90 or above (10)\n over 1000 (0)\n", dsr.Facets.GetTopChildren(10, "field").ToString()); // Second search, drill down on dim=b: ddq = new DrillDownQuery(config); ddq.Add("dim", "b"); dsr = ds.Search(null, ddq, 10); Assert.AreEqual(75, dsr.Hits.TotalHits); Assert.AreEqual("dim=dim path=[] value=100 childCount=2\n b (75)\n a (25)\n", dsr.Facets.GetTopChildren(10, "dim").ToString()); Assert.AreEqual("dim=field path=[] value=16 childCount=5\n less than 10 (7)\n less than or equal to 10 (8)\n over 90 (7)\n 90 or above (8)\n over 1000 (0)\n", dsr.Facets.GetTopChildren(10, "field").ToString()); // Third search, drill down on "less than or equal to 10": ddq = new DrillDownQuery(config); ddq.Add("field", NumericRangeQuery.NewInt64Range("field", 0L, 10L, true, true)); dsr = ds.Search(null, ddq, 10); Assert.AreEqual(11, dsr.Hits.TotalHits); Assert.AreEqual("dim=dim path=[] value=11 childCount=2\n b (8)\n a (3)\n", dsr.Facets.GetTopChildren(10, "dim").ToString()); Assert.AreEqual("dim=field path=[] value=21 childCount=5\n less than 10 (10)\n less than or equal to 10 (11)\n over 90 (9)\n 90 or above (10)\n over 1000 (0)\n", dsr.Facets.GetTopChildren(10, "field").ToString()); IOUtils.Dispose(tw, tr, td, w, r, d); }
public virtual void TestL2OBasic() { LabelToOrdinal map = new LabelToOrdinalMap(); CompactLabelToOrdinal compact = new CompactLabelToOrdinal(200, 0.15f, 3); int n = 50; string[] uniqueValues = new string[] { //@"�", //@"�r�G��F�\u0382�7\u0019�h�\u0015���#\u001d3\r{��q�_���Ԃ������", "foo bar one", //new string(new char[] { (char)65533, (char)65533, (char)65, (char)65533, (char)45, (char)106, (char)40, (char)643, (char)65533, (char)11, (char)65533, (char)88, (char)65533, (char)78, (char)126, (char)56, (char)12, (char)71 }), //"foo bar two", //"foo bar three", //"foo bar four", //"foo bar five", //"foo bar six", //"foo bar seven", //"foo bar eight", //"foo bar nine", //"foo bar ten", //"foo/bar/one", //"foo/bar/two", //"foo/bar/three", //"foo/bar/four", //"foo/bar/five", //"foo/bar/six", //"foo/bar/seven", //"foo/bar/eight", //"foo/bar/nine", //"foo/bar/ten", //"" }; var tmpDir = CreateTempDir("testLableToOrdinal"); var f = new FileInfo(Path.Combine(tmpDir.FullName, "CompactLabelToOrdinalTest.tmp")); int flushInterval = 10; for (int i = 0; i < n; i++) { if (i > 0 && i % flushInterval == 0) { using (var fileStream = new FileStream(f.FullName, FileMode.OpenOrCreate, FileAccess.ReadWrite)) { compact.Flush(fileStream); } compact = CompactLabelToOrdinal.Open(f, 0.15f, 3); //assertTrue(f.Delete()); f.Delete(); assertFalse(File.Exists(f.FullName)); if (flushInterval < (n / 10)) { flushInterval *= 10; } } FacetLabel label = new FacetLabel(); foreach (string s in uniqueValues) { if (s.Length == 0) { label = new FacetLabel(); } else { label = new FacetLabel(s.Split(new char[] { '/' }, StringSplitOptions.RemoveEmptyEntries)); } int ord1 = map.GetOrdinal(label); int ord2 = compact.GetOrdinal(label); if (Verbose) { Console.WriteLine("Testing label: " + label.ToString()); } assertEquals(ord1, ord2); if (ord1 == LabelToOrdinal.INVALID_ORDINAL) { ord1 = compact.GetNextOrdinal(); map.AddLabel(label, ord1); compact.AddLabel(label, ord1); } } } for (int i = 0; i < uniqueValues.Length; i++) { FacetLabel label; string s = uniqueValues[i]; if (s.Length == 0) { label = new FacetLabel(); } else { label = new FacetLabel(s.Split(new char[] { '/' }, StringSplitOptions.RemoveEmptyEntries)); } int ord1 = map.GetOrdinal(label); int ord2 = compact.GetOrdinal(label); if (Verbose) { Console.WriteLine("Testing label 2: " + label.ToString()); } assertEquals(ord1, ord2); } }
public virtual void TestL2O() { LabelToOrdinal map = new LabelToOrdinalMap(); CompactLabelToOrdinal compact = new CompactLabelToOrdinal(2000000, 0.15f, 3); int n = AtLeast(10 * 1000); const int numUniqueValues = 50 * 1000; string[] uniqueValues = new string[numUniqueValues]; byte[] buffer = new byte[50]; // This is essentially the equivalent of // CharsetDecoder decoder = StandardCharsets.UTF_8.newDecoder() // .onUnmappableCharacter(CodingErrorAction.REPLACE) // .onMalformedInput(CodingErrorAction.REPLACE); // // Encoding decoder = Encoding.GetEncoding(Encoding.UTF8.CodePage, // new EncoderReplacementFallback("?"), // new DecoderReplacementFallback("?")); Random random = Random; for (int i = 0; i < numUniqueValues;) { random.NextBytes(buffer); int size = 1 + random.Next(buffer.Length); // This test is turning random bytes into a string, // this is asking for trouble. Encoding decoder = Encoding.GetEncoding(Encoding.UTF8.CodePage, new EncoderReplacementFallback("?"), new DecoderReplacementFallback("?")); uniqueValues[i] = decoder.GetString(buffer, 0, size); // we cannot have empty path components, so eliminate all prefix as well // as middle consecutive delimiter chars. uniqueValues[i] = Regex.Replace(uniqueValues[i], "/+", "/"); if (uniqueValues[i].StartsWith("/", StringComparison.Ordinal)) { uniqueValues[i] = uniqueValues[i].Substring(1); } if (uniqueValues[i].IndexOf(CompactLabelToOrdinal.TERMINATOR_CHAR) == -1) { i++; } } var tmpDir = CreateTempDir("testLableToOrdinal"); var f = new FileInfo(Path.Combine(tmpDir.FullName, "CompactLabelToOrdinalTest.tmp")); int flushInterval = 10; for (int i = 0; i < n; i++) { if (i > 0 && i % flushInterval == 0) { using (var fileStream = new FileStream(f.FullName, FileMode.OpenOrCreate, FileAccess.ReadWrite)) { compact.Flush(fileStream); } compact = CompactLabelToOrdinal.Open(f, 0.15f, 3); //assertTrue(f.Delete()); f.Delete(); assertFalse(File.Exists(f.FullName)); if (flushInterval < (n / 10)) { flushInterval *= 10; } } int index = random.Next(numUniqueValues); FacetLabel label; string s = uniqueValues[index]; if (s.Length == 0) { label = new FacetLabel(); } else { label = new FacetLabel(s.Split(new char[] { '/' }, StringSplitOptions.RemoveEmptyEntries)); } int ord1 = map.GetOrdinal(label); int ord2 = compact.GetOrdinal(label); if (Verbose) { Console.WriteLine("Testing label: " + label.ToString()); } assertEquals(ord1, ord2); if (ord1 == LabelToOrdinal.INVALID_ORDINAL) { ord1 = compact.GetNextOrdinal(); map.AddLabel(label, ord1); compact.AddLabel(label, ord1); } } for (int i = 0; i < numUniqueValues; i++) { FacetLabel label; string s = uniqueValues[i]; if (s.Length == 0) { label = new FacetLabel(); } else { label = new FacetLabel(s.Split(new char[] { '/' }, StringSplitOptions.RemoveEmptyEntries)); } int ord1 = map.GetOrdinal(label); int ord2 = compact.GetOrdinal(label); if (Verbose) { Console.WriteLine("Testing label 2: " + label.ToString()); } assertEquals(ord1, ord2); } }
private void Verify(AtomicReader r, int[][] idToOrds, BytesRef[] termsArray, BytesRef prefixRef) { DocTermOrds dto = new DocTermOrds(r, r.LiveDocs, "field", prefixRef, int.MaxValue, TestUtil.NextInt32(Random, 2, 10)); FieldCache.Int32s docIDToID = FieldCache.DEFAULT.GetInt32s(r, "id", false); /* * for(int docID=0;docID<subR.MaxDoc;docID++) { * System.out.println(" docID=" + docID + " id=" + docIDToID[docID]); * } */ if (Verbose) { Console.WriteLine("TEST: verify prefix=" + (prefixRef == null ? "null" : prefixRef.Utf8ToString())); Console.WriteLine("TEST: all TERMS:"); TermsEnum allTE = MultiFields.GetTerms(r, "field").GetIterator(null); int ord = 0; while (allTE.Next() != null) { Console.WriteLine(" ord=" + (ord++) + " term=" + allTE.Term.Utf8ToString()); } } //final TermsEnum te = subR.Fields.Terms("field").iterator(); TermsEnum te = dto.GetOrdTermsEnum(r); if (dto.NumTerms == 0) { if (prefixRef == null) { Assert.IsNull(MultiFields.GetTerms(r, "field")); } else { Terms terms = MultiFields.GetTerms(r, "field"); if (terms != null) { TermsEnum termsEnum = terms.GetIterator(null); TermsEnum.SeekStatus result = termsEnum.SeekCeil(prefixRef); if (result != TermsEnum.SeekStatus.END) { Assert.IsFalse(StringHelper.StartsWith(termsEnum.Term, prefixRef), "term=" + termsEnum.Term.Utf8ToString() + " matches prefix=" + prefixRef.Utf8ToString()); } else { // ok } } else { // ok } } return; } if (Verbose) { Console.WriteLine("TEST: TERMS:"); te.SeekExact(0); while (true) { Console.WriteLine(" ord=" + te.Ord + " term=" + te.Term.Utf8ToString()); if (te.Next() == null) { break; } } } SortedSetDocValues iter = dto.GetIterator(r); for (int docID = 0; docID < r.MaxDoc; docID++) { if (Verbose) { Console.WriteLine("TEST: docID=" + docID + " of " + r.MaxDoc + " (id=" + docIDToID.Get(docID) + ")"); } iter.SetDocument(docID); int[] answers = idToOrds[docIDToID.Get(docID)]; int upto = 0; long ord; while ((ord = iter.NextOrd()) != SortedSetDocValues.NO_MORE_ORDS) { te.SeekExact(ord); BytesRef expected = termsArray[answers[upto++]]; if (Verbose) { Console.WriteLine(" exp=" + expected.Utf8ToString() + " actual=" + te.Term.Utf8ToString()); } Assert.AreEqual(expected, te.Term, "expected=" + expected.Utf8ToString() + " actual=" + te.Term.Utf8ToString() + " ord=" + ord); } Assert.AreEqual(answers.Length, upto); } }
public virtual void TestRandomWithPrefix() { Directory dir = NewDirectory(); ISet <string> prefixes = new JCG.HashSet <string>(); int numPrefix = TestUtil.NextInt32(Random, 2, 7); if (Verbose) { Console.WriteLine("TEST: use " + numPrefix + " prefixes"); } while (prefixes.Count < numPrefix) { prefixes.Add(TestUtil.RandomRealisticUnicodeString(Random)); //prefixes.Add(TestUtil.RandomSimpleString(random)); } string[] prefixesArray = prefixes.ToArray(/*new string[prefixes.Count]*/); int NUM_TERMS = AtLeast(20); ISet <BytesRef> terms = new JCG.HashSet <BytesRef>(); while (terms.Count < NUM_TERMS) { string s = prefixesArray[Random.Next(prefixesArray.Length)] + TestUtil.RandomRealisticUnicodeString(Random); //final String s = prefixesArray[random.nextInt(prefixesArray.Length)] + TestUtil.RandomSimpleString(random); if (s.Length > 0) { terms.Add(new BytesRef(s)); } } BytesRef[] termsArray = terms.ToArray(); Array.Sort(termsArray); int NUM_DOCS = AtLeast(100); IndexWriterConfig conf = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random)); // Sometimes swap in codec that impls ord(): if (Random.Next(10) == 7) { Codec codec = TestUtil.AlwaysPostingsFormat(PostingsFormat.ForName("Lucene41WithOrds")); conf.SetCodec(codec); } RandomIndexWriter w = new RandomIndexWriter(Random, dir, conf); int[][] idToOrds = new int[NUM_DOCS][]; ISet <int?> ordsForDocSet = new JCG.HashSet <int?>(); for (int id = 0; id < NUM_DOCS; id++) { Document doc = new Document(); doc.Add(new Int32Field("id", id, Field.Store.NO)); int termCount = TestUtil.NextInt32(Random, 0, 20 * RandomMultiplier); while (ordsForDocSet.Count < termCount) { ordsForDocSet.Add(Random.Next(termsArray.Length)); } int[] ordsForDoc = new int[termCount]; int upto = 0; if (Verbose) { Console.WriteLine("TEST: doc id=" + id); } foreach (int ord in ordsForDocSet) { ordsForDoc[upto++] = ord; Field field = NewStringField("field", termsArray[ord].Utf8ToString(), Field.Store.NO); if (Verbose) { Console.WriteLine(" f=" + termsArray[ord].Utf8ToString()); } doc.Add(field); } ordsForDocSet.Clear(); Array.Sort(ordsForDoc); idToOrds[id] = ordsForDoc; w.AddDocument(doc); } DirectoryReader r = w.GetReader(); w.Dispose(); if (Verbose) { Console.WriteLine("TEST: reader=" + r); } AtomicReader slowR = SlowCompositeReaderWrapper.Wrap(r); foreach (string prefix in prefixesArray) { BytesRef prefixRef = prefix == null ? null : new BytesRef(prefix); int[][] idToOrdsPrefix = new int[NUM_DOCS][]; for (int id = 0; id < NUM_DOCS; id++) { int[] docOrds = idToOrds[id]; IList <int?> newOrds = new List <int?>(); foreach (int ord in idToOrds[id]) { if (StringHelper.StartsWith(termsArray[ord], prefixRef)) { newOrds.Add(ord); } } int[] newOrdsArray = new int[newOrds.Count]; int upto = 0; foreach (int ord in newOrds) { newOrdsArray[upto++] = ord; } idToOrdsPrefix[id] = newOrdsArray; } foreach (AtomicReaderContext ctx in r.Leaves) { if (Verbose) { Console.WriteLine("\nTEST: sub=" + ctx.Reader); } Verify((AtomicReader)ctx.Reader, idToOrdsPrefix, termsArray, prefixRef); } // Also test top-level reader: its enum does not support // ord, so this forces the OrdWrapper to run: if (Verbose) { Console.WriteLine("TEST: top reader"); } Verify(slowR, idToOrdsPrefix, termsArray, prefixRef); } FieldCache.DEFAULT.PurgeByCacheKey(slowR.CoreCacheKey); r.Dispose(); dir.Dispose(); }
public virtual void TestAccquireReleaseRace() { DocumentsWriterStallControl ctrl = new DocumentsWriterStallControl(); ctrl.UpdateStalled(false); AtomicBoolean stop = new AtomicBoolean(false); AtomicBoolean checkPoint = new AtomicBoolean(true); int numStallers = AtLeast(1); int numReleasers = AtLeast(1); int numWaiters = AtLeast(1); var sync = new Synchronizer(numStallers + numReleasers, numStallers + numReleasers + numWaiters); var threads = new ThreadJob[numReleasers + numStallers + numWaiters]; IList <Exception> exceptions = new SynchronizedList <Exception>(); for (int i = 0; i < numReleasers; i++) { threads[i] = new Updater(stop, checkPoint, ctrl, sync, true, exceptions); } for (int i = numReleasers; i < numReleasers + numStallers; i++) { threads[i] = new Updater(stop, checkPoint, ctrl, sync, false, exceptions); } for (int i = numReleasers + numStallers; i < numReleasers + numStallers + numWaiters; i++) { threads[i] = new Waiter(stop, checkPoint, ctrl, sync, exceptions); } Start(threads); int iters = AtLeast(10000); float checkPointProbability = TestNightly ? 0.5f : 0.1f; for (int i = 0; i < iters; i++) { if (checkPoint) { Assert.IsTrue(sync.updateJoin.Wait(new TimeSpan(0, 0, 0, 10)), "timed out waiting for update threads - deadlock?"); if (exceptions.Count > 0) { foreach (Exception throwable in exceptions) { Console.WriteLine(throwable.ToString()); Console.Write(throwable.StackTrace); } Assert.Fail("got exceptions in threads"); } if (ctrl.HasBlocked && ctrl.IsHealthy) { AssertState(numReleasers, numStallers, numWaiters, threads, ctrl); } checkPoint.Value = (false); sync.waiter.Signal(); sync.leftCheckpoint.Wait(); } Assert.IsFalse(checkPoint); Assert.AreEqual(0, sync.waiter.CurrentCount); if (checkPointProbability >= (float)Random.NextDouble()) { sync.Reset(numStallers + numReleasers, numStallers + numReleasers + numWaiters); checkPoint.Value = (true); } } if (!checkPoint) { sync.Reset(numStallers + numReleasers, numStallers + numReleasers + numWaiters); checkPoint.Value = (true); } Assert.IsTrue(sync.updateJoin.Wait(new TimeSpan(0, 0, 0, 10))); AssertState(numReleasers, numStallers, numWaiters, threads, ctrl); checkPoint.Value = (false); stop.Value = (true); sync.waiter.Signal(); sync.leftCheckpoint.Wait(); for (int i = 0; i < threads.Length; i++) { ctrl.UpdateStalled(false); threads[i].Join(2000); if (threads[i].IsAlive && threads[i] is Waiter) { if (threads[i].State == ThreadState.WaitSleepJoin) { Assert.Fail("waiter is not released - anyThreadsStalled: " + ctrl.AnyStalledThreads()); } } } }
public virtual void TestNoWaitClose() { Directory directory = NewDirectory(); Document doc = new Document(); FieldType customType = new FieldType(TextField.TYPE_STORED); customType.IsTokenized = false; Field idField = NewField("id", "", customType); doc.Add(idField); for (int pass = 0; pass < 2; pass++) { if (Verbose) { Console.WriteLine("TEST: pass="******"TEST: iter=" + iter); } for (int j = 0; j < 199; j++) { idField.SetStringValue(Convert.ToString(iter * 201 + j)); writer.AddDocument(doc); } int delID = iter * 199; for (int j = 0; j < 20; j++) { writer.DeleteDocuments(new Term("id", Convert.ToString(delID))); delID += 5; } // Force a bunch of merge threads to kick off so we // stress out aborting them on close: ((LogMergePolicy)writer.Config.MergePolicy).MergeFactor = 2; IndexWriter finalWriter = writer; List <Exception> failure = new List <Exception>(); ThreadJob t1 = new ThreadAnonymousInnerClassHelper(this, doc, finalWriter, failure); if (failure.Count > 0) { throw failure[0]; } t1.Start(); writer.Dispose(false); t1.Join(); // Make sure reader can read IndexReader reader = DirectoryReader.Open(directory); reader.Dispose(); // Reopen writer = new IndexWriter(directory, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random)).SetOpenMode(OpenMode.APPEND).SetMergePolicy(NewLogMergePolicy())); } writer.Dispose(); } directory.Dispose(); }
public virtual void TestRandomDoubles() { Directory dir = NewDirectory(); RandomIndexWriter w = new RandomIndexWriter( #if FEATURE_INSTANCE_TESTDATA_INITIALIZATION this, #endif Random, dir); int numDocs = AtLeast(1000); double[] values = new double[numDocs]; double minValue = double.PositiveInfinity; double maxValue = double.NegativeInfinity; for (int i = 0; i < numDocs; i++) { Document doc = new Document(); double v = Random.NextDouble(); values[i] = v; doc.Add(new DoubleDocValuesField("field", v)); doc.Add(new DoubleField("field", v, Field.Store.NO)); w.AddDocument(doc); minValue = Math.Min(minValue, v); maxValue = Math.Max(maxValue, v); } IndexReader r = w.GetReader(); IndexSearcher s = NewSearcher(r); FacetsConfig config = new FacetsConfig(); int numIters = AtLeast(10); for (int iter = 0; iter < numIters; iter++) { if (VERBOSE) { Console.WriteLine("TEST: iter=" + iter); } int numRange = TestUtil.NextInt32(Random, 1, 5); DoubleRange[] ranges = new DoubleRange[numRange]; int[] expectedCounts = new int[numRange]; double minAcceptedValue = double.PositiveInfinity; double maxAcceptedValue = double.NegativeInfinity; for (int rangeID = 0; rangeID < numRange; rangeID++) { double min; if (rangeID > 0 && Random.Next(10) == 7) { // Use an existing boundary: DoubleRange prevRange = ranges[Random.Next(rangeID)]; if (Random.NextBoolean()) { min = prevRange.Min; } else { min = prevRange.Max; } } else { min = Random.NextDouble(); } double max; if (rangeID > 0 && Random.Next(10) == 7) { // Use an existing boundary: DoubleRange prevRange = ranges[Random.Next(rangeID)]; if (Random.NextBoolean()) { max = prevRange.Min; } else { max = prevRange.Max; } } else { max = Random.NextDouble(); } if (min > max) { double x = min; min = max; max = x; } bool minIncl; bool maxIncl; if (min == max) { minIncl = true; maxIncl = true; } else { minIncl = Random.NextBoolean(); maxIncl = Random.NextBoolean(); } ranges[rangeID] = new DoubleRange("r" + rangeID, min, minIncl, max, maxIncl); // Do "slow but hopefully correct" computation of // expected count: for (int i = 0; i < numDocs; i++) { bool accept = true; if (minIncl) { accept &= values[i] >= min; } else { accept &= values[i] > min; } if (maxIncl) { accept &= values[i] <= max; } else { accept &= values[i] < max; } if (accept) { expectedCounts[rangeID]++; minAcceptedValue = Math.Min(minAcceptedValue, values[i]); maxAcceptedValue = Math.Max(maxAcceptedValue, values[i]); } } } FacetsCollector sfc = new FacetsCollector(); s.Search(new MatchAllDocsQuery(), sfc); Filter fastMatchFilter; if (Random.NextBoolean()) { if (Random.NextBoolean()) { fastMatchFilter = NumericRangeFilter.NewDoubleRange("field", minValue, maxValue, true, true); } else { fastMatchFilter = NumericRangeFilter.NewDoubleRange("field", minAcceptedValue, maxAcceptedValue, true, true); } } else { fastMatchFilter = null; } ValueSource vs = new DoubleFieldSource("field"); Facets facets = new DoubleRangeFacetCounts("field", vs, sfc, fastMatchFilter, ranges); FacetResult result = facets.GetTopChildren(10, "field"); Assert.AreEqual(numRange, result.LabelValues.Length); for (int rangeID = 0; rangeID < numRange; rangeID++) { if (VERBOSE) { Console.WriteLine(" range " + rangeID + " expectedCount=" + expectedCounts[rangeID]); } LabelAndValue subNode = result.LabelValues[rangeID]; Assert.AreEqual("r" + rangeID, subNode.Label); Assert.AreEqual(expectedCounts[rangeID], (int)subNode.Value); DoubleRange range = ranges[rangeID]; // Test drill-down: DrillDownQuery ddq = new DrillDownQuery(config); if (Random.NextBoolean()) { if (Random.NextBoolean()) { ddq.Add("field", NumericRangeFilter.NewDoubleRange("field", range.Min, range.Max, range.MinInclusive, range.MaxInclusive)); } else { ddq.Add("field", NumericRangeQuery.NewDoubleRange("field", range.Min, range.Max, range.MinInclusive, range.MaxInclusive)); } } else { ddq.Add("field", range.GetFilter(fastMatchFilter, vs)); } Assert.AreEqual(expectedCounts[rangeID], s.Search(ddq, 10).TotalHits); } } IOUtils.Dispose(w, r, dir); }
public virtual void TestConcurrency() { int ncats = AtLeast(100000); // add many categories int range = ncats * 3; // affects the categories selection AtomicInt32 numCats = new AtomicInt32(ncats); Directory dir = NewDirectory(); var values = new ConcurrentDictionary <string, string>(); double d = Random.NextDouble(); ITaxonomyWriterCache cache; if (d < 0.7) { // this is the fastest, yet most memory consuming cache = new Cl2oTaxonomyWriterCache(1024, 0.15f, 3); } // LUCENENET specific - this option takes too long to get under the 1 hour job limit in Azure DevOps //else if (TestNightly && d > 0.98) //{ // // this is the slowest, but tests the writer concurrency when no caching is done. // // only pick it during NIGHTLY tests, and even then, with very low chances. // cache = NO_OP_CACHE; //} else { // this is slower than CL2O, but less memory consuming, and exercises finding categories on disk too. cache = new LruTaxonomyWriterCache(ncats / 10); } if (Verbose) { Console.WriteLine("TEST: use cache=" + cache); } var tw = new DirectoryTaxonomyWriter(dir, OpenMode.CREATE, cache); ThreadJob[] addThreads = new ThreadJob[AtLeast(4)]; for (int z = 0; z < addThreads.Length; z++) { addThreads[z] = new ThreadAnonymousClass(range, numCats, values, tw); } foreach (var t in addThreads) { t.Start(); } foreach (var t in addThreads) { t.Join(); } tw.Dispose(); DirectoryTaxonomyReader dtr = new DirectoryTaxonomyReader(dir); // +1 for root category if (values.Count + 1 != dtr.Count) { foreach (string value in values.Keys) { FacetLabel label = new FacetLabel(FacetsConfig.StringToPath(value)); if (dtr.GetOrdinal(label) == -1) { Console.WriteLine("FAIL: path=" + label + " not recognized"); } } fail("mismatch number of categories"); } int[] parents = dtr.ParallelTaxonomyArrays.Parents; foreach (string cat in values.Keys) { FacetLabel cp = new FacetLabel(FacetsConfig.StringToPath(cat)); Assert.IsTrue(dtr.GetOrdinal(cp) > 0, "category not found " + cp); int level = cp.Length; int parentOrd = 0; // for root, parent is always virtual ROOT (ord=0) FacetLabel path /*= new FacetLabel()*/; for (int i = 0; i < level; i++) { path = cp.Subpath(i + 1); int ord = dtr.GetOrdinal(path); Assert.AreEqual(parentOrd, parents[ord], "invalid parent for cp=" + path); parentOrd = ord; // next level should have this parent } } IOUtils.Dispose(dtr, dir); }
public virtual void TestNrt() { Store.Directory dir = NewDirectory(); Store.Directory taxoDir = NewDirectory(); IndexWriterConfig iwc = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random)); // Don't allow tiny maxBufferedDocs; it can make this // test too slow: iwc.SetMaxBufferedDocs(Math.Max(500, iwc.MaxBufferedDocs)); // MockRandom/AlcololicMergePolicy are too slow: TieredMergePolicy tmp = new TieredMergePolicy(); tmp.FloorSegmentMB = .001; iwc.SetMergePolicy(tmp); IndexWriter w = new IndexWriter(dir, iwc); var tw = new DirectoryTaxonomyWriter(taxoDir); FacetsConfig config = new FacetsConfig(); config.SetMultiValued("field", true); AtomicBoolean stop = new AtomicBoolean(); // How many unique facets to index before stopping: //int ordLimit = TestNightly ? 100000 : 6000; // LUCENENET specific: 100000 facets takes about 2-3 hours. To keep it under // the 1 hour free limit of Azure DevOps, this was reduced to 30000. int ordLimit = TestNightly ? 30000 : 6000; var indexer = new IndexerThread(w, config, tw, null, ordLimit, stop); var mgr = new SearcherTaxonomyManager(w, true, null, tw); var reopener = new ThreadAnonymousInnerClassHelper(stop, mgr); reopener.Name = "reopener"; reopener.Start(); indexer.Name = "indexer"; indexer.Start(); try { while (!stop) { SearcherAndTaxonomy pair = mgr.Acquire(); try { //System.out.println("search maxOrd=" + pair.taxonomyReader.getSize()); FacetsCollector sfc = new FacetsCollector(); pair.Searcher.Search(new MatchAllDocsQuery(), sfc); Facets facets = GetTaxonomyFacetCounts(pair.TaxonomyReader, config, sfc); FacetResult result = facets.GetTopChildren(10, "field"); if (pair.Searcher.IndexReader.NumDocs > 0) { //System.out.println(pair.taxonomyReader.getSize()); Assert.IsTrue(result.ChildCount > 0); Assert.IsTrue(result.LabelValues.Length > 0); } //if (VERBOSE) { //System.out.println("TEST: facets=" + FacetTestUtils.toString(results.get(0))); //} } finally { mgr.Release(pair); } } } finally { indexer.Join(); reopener.Join(); } if (Verbose) { Console.WriteLine("TEST: now stop"); } IOUtils.Dispose(mgr, tw, w, taxoDir, dir); }
public virtual void Test() { Random random = new Random(Random.Next()); LineFileDocs docs = new LineFileDocs(random, DefaultCodecSupportsDocValues); Directory d = NewDirectory(); MockAnalyzer analyzer = new MockAnalyzer(LuceneTestCase.Random); analyzer.MaxTokenLength = TestUtil.NextInt32(LuceneTestCase.Random, 1, IndexWriter.MAX_TERM_LENGTH); RandomIndexWriter w = new RandomIndexWriter( #if FEATURE_INSTANCE_TESTDATA_INITIALIZATION this, #endif LuceneTestCase.Random, d, analyzer); int numDocs = AtLeast(10); for (int docCount = 0; docCount < numDocs; docCount++) { w.AddDocument(docs.NextDoc()); } IndexReader r = w.GetReader(); w.Dispose(); List <BytesRef> terms = new List <BytesRef>(); TermsEnum termsEnum = MultiFields.GetTerms(r, "body").GetIterator(null); BytesRef term; while ((term = termsEnum.Next()) != null) { terms.Add(BytesRef.DeepCopyOf(term)); } if (VERBOSE) { Console.WriteLine("TEST: " + terms.Count + " terms"); } int upto = -1; int iters = AtLeast(200); for (int iter = 0; iter < iters; iter++) { bool isEnd; if (upto != -1 && LuceneTestCase.Random.NextBoolean()) { // next if (VERBOSE) { Console.WriteLine("TEST: iter next"); } isEnd = termsEnum.Next() == null; upto++; if (isEnd) { if (VERBOSE) { Console.WriteLine(" end"); } Assert.AreEqual(upto, terms.Count); upto = -1; } else { if (VERBOSE) { Console.WriteLine(" got term=" + termsEnum.Term.Utf8ToString() + " expected=" + terms[upto].Utf8ToString()); } Assert.IsTrue(upto < terms.Count); Assert.AreEqual(terms[upto], termsEnum.Term); } } else { BytesRef target; string exists; if (LuceneTestCase.Random.NextBoolean()) { // likely fake term if (LuceneTestCase.Random.NextBoolean()) { target = new BytesRef(TestUtil.RandomSimpleString(LuceneTestCase.Random)); } else { target = new BytesRef(TestUtil.RandomRealisticUnicodeString(LuceneTestCase.Random)); } exists = "likely not"; } else { // real term target = terms[LuceneTestCase.Random.Next(terms.Count)]; exists = "yes"; } upto = terms.BinarySearch(target); if (LuceneTestCase.Random.NextBoolean()) { if (VERBOSE) { Console.WriteLine("TEST: iter seekCeil target=" + target.Utf8ToString() + " exists=" + exists); } // seekCeil TermsEnum.SeekStatus status = termsEnum.SeekCeil(target); if (VERBOSE) { Console.WriteLine(" got " + status); } if (upto < 0) { upto = -(upto + 1); if (upto >= terms.Count) { Assert.AreEqual(TermsEnum.SeekStatus.END, status); upto = -1; } else { Assert.AreEqual(TermsEnum.SeekStatus.NOT_FOUND, status); Assert.AreEqual(terms[upto], termsEnum.Term); } } else { Assert.AreEqual(TermsEnum.SeekStatus.FOUND, status); Assert.AreEqual(terms[upto], termsEnum.Term); } } else { if (VERBOSE) { Console.WriteLine("TEST: iter seekExact target=" + target.Utf8ToString() + " exists=" + exists); } // seekExact bool result = termsEnum.SeekExact(target); if (VERBOSE) { Console.WriteLine(" got " + result); } if (upto < 0) { Assert.IsFalse(result); upto = -1; } else { Assert.IsTrue(result); Assert.AreEqual(target, termsEnum.Term); } } } } r.Dispose(); d.Dispose(); docs.Dispose(); }
public virtual void Test_Directory() // LUCENENET specific - name collides with property of LuceneTestCase { Store.Directory indexDir = NewDirectory(); Store.Directory taxoDir = NewDirectory(); IndexWriter w = new IndexWriter(indexDir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random))); var tw = new DirectoryTaxonomyWriter(taxoDir); // first empty commit w.Commit(); tw.Commit(); var mgr = new SearcherTaxonomyManager(indexDir, taxoDir, null); FacetsConfig config = new FacetsConfig(); config.SetMultiValued("field", true); AtomicBoolean stop = new AtomicBoolean(); // How many unique facets to index before stopping: //int ordLimit = TestNightly ? 100000 : 6000; // LUCENENET specific: 100000 facets takes about 2-3 hours. To keep it under // the 1 hour free limit of Azure DevOps, this was reduced to 30000. int ordLimit = TestNightly ? 30000 : 6000; var indexer = new IndexerThread(w, config, tw, mgr, ordLimit, stop); indexer.Start(); try { while (!stop) { SearcherAndTaxonomy pair = mgr.Acquire(); try { //System.out.println("search maxOrd=" + pair.taxonomyReader.getSize()); FacetsCollector sfc = new FacetsCollector(); pair.Searcher.Search(new MatchAllDocsQuery(), sfc); Facets facets = GetTaxonomyFacetCounts(pair.TaxonomyReader, config, sfc); FacetResult result = facets.GetTopChildren(10, "field"); if (pair.Searcher.IndexReader.NumDocs > 0) { //System.out.println(pair.taxonomyReader.getSize()); Assert.IsTrue(result.ChildCount > 0); Assert.IsTrue(result.LabelValues.Length > 0); } //if (VERBOSE) { //System.out.println("TEST: facets=" + FacetTestUtils.toString(results.get(0))); //} } finally { mgr.Release(pair); } } } finally { indexer.Join(); } if (Verbose) { Console.WriteLine("TEST: now stop"); } IOUtils.Dispose(mgr, tw, w, taxoDir, indexDir); }
private void Verify(BytesStore bytes, byte[] expected, int totalLength) { Assert.AreEqual(totalLength, bytes.Position); if (totalLength == 0) { return; } if (VERBOSE) { Console.WriteLine(" verify..."); } // First verify whole thing in one blast: byte[] actual = new byte[totalLength]; if (Random.NextBoolean()) { if (VERBOSE) { Console.WriteLine(" bulk: reversed"); } // reversed FST.BytesReader r2 = bytes.GetReverseReader(); Assert.IsTrue(r2.IsReversed); r2.Position = totalLength - 1; r2.ReadBytes(actual, 0, actual.Length); int start = 0; int end = totalLength - 1; while (start < end) { byte b = actual[start]; actual[start] = actual[end]; actual[end] = b; start++; end--; } } else { // forward if (VERBOSE) { Console.WriteLine(" bulk: forward"); } FST.BytesReader r3 = bytes.GetForwardReader(); Assert.IsFalse(r3.IsReversed); r3.ReadBytes(actual, 0, actual.Length); } for (int i = 0; i < totalLength; i++) { assertEquals("byte @ index=" + i, expected[i], actual[i]); } FST.BytesReader r; // Then verify ops: bool reversed = Random.NextBoolean(); if (reversed) { if (VERBOSE) { Console.WriteLine(" ops: reversed"); } r = bytes.GetReverseReader(); } else { if (VERBOSE) { Console.WriteLine(" ops: forward"); } r = bytes.GetForwardReader(); } if (totalLength > 1) { int numOps = TestUtil.NextInt32(Random, 100, 200); for (int op = 0; op < numOps; op++) { int numBytes = Random.Next(Math.Min(1000, totalLength - 1)); int pos; if (reversed) { pos = TestUtil.NextInt32(Random, numBytes, totalLength - 1); } else { pos = Random.Next(totalLength - numBytes); } if (VERBOSE) { Console.WriteLine(" op iter=" + op + " reversed=" + reversed + " numBytes=" + numBytes + " pos=" + pos); } byte[] temp = new byte[numBytes]; r.Position = pos; Assert.AreEqual(pos, r.Position); r.ReadBytes(temp, 0, temp.Length); for (int i = 0; i < numBytes; i++) { byte expectedByte; if (reversed) { expectedByte = expected[pos - i]; } else { expectedByte = expected[pos + i]; } assertEquals("byte @ index=" + i, expectedByte, temp[i]); } int left; int expectedPos; if (reversed) { expectedPos = pos - numBytes; left = (int)r.Position; } else { expectedPos = pos + numBytes; left = (int)(totalLength - r.Position); } Assert.AreEqual(expectedPos, r.Position); if (left > 4) { int skipBytes = Random.Next(left - 4); int expectedInt = 0; if (reversed) { expectedPos -= skipBytes; expectedInt |= (expected[expectedPos--] & 0xFF) << 24; expectedInt |= (expected[expectedPos--] & 0xFF) << 16; expectedInt |= (expected[expectedPos--] & 0xFF) << 8; expectedInt |= (expected[expectedPos--] & 0xFF); } else { expectedPos += skipBytes; expectedInt |= (expected[expectedPos++] & 0xFF) << 24; expectedInt |= (expected[expectedPos++] & 0xFF) << 16; expectedInt |= (expected[expectedPos++] & 0xFF) << 8; expectedInt |= (expected[expectedPos++] & 0xFF); } if (VERBOSE) { Console.WriteLine(" skip numBytes=" + skipBytes); Console.WriteLine(" readInt"); } r.SkipBytes(skipBytes); Assert.AreEqual(expectedInt, r.ReadInt32()); } } } }
public override void Run() { try { var seen = new JCG.HashSet <string>(); IList <string> paths = new List <string>(); while (true) { Document doc = new Document(); int numPaths = TestUtil.NextInt32(Random, 1, 5); for (int i = 0; i < numPaths; i++) { string path; if (paths.Count > 0 && Random.Next(5) != 4) { // Use previous path path = paths[Random.Next(paths.Count)]; } else { // Create new path path = null; while (true) { path = TestUtil.RandomRealisticUnicodeString(Random); if (path.Length != 0 && !seen.Contains(path)) { seen.Add(path); paths.Add(path); break; } } } doc.Add(new FacetField("field", path)); } try { w.AddDocument(config.Build(tw, doc)); if (mgr != null && Random.NextDouble() < 0.02) { w.Commit(); tw.Commit(); mgr.MaybeRefresh(); } } catch (IOException ioe) { throw new Exception(ioe.ToString(), ioe); } if (Verbose) { Console.WriteLine("TW size=" + tw.Count + " vs " + ordLimit); } if (tw.Count >= ordLimit) { break; } } } finally { stop.Value = true; } }
public virtual void TestRandomStringSort() { Random random = new Random(Random.Next()); int NUM_DOCS = AtLeast(100); Directory dir = NewDirectory(); RandomIndexWriter writer = new RandomIndexWriter( #if FEATURE_INSTANCE_TESTDATA_INITIALIZATION this, #endif random, dir); bool allowDups = random.NextBoolean(); ISet <string> seen = new JCG.HashSet <string>(); int maxLength = TestUtil.NextInt32(random, 5, 100); if (Verbose) { Console.WriteLine("TEST: NUM_DOCS=" + NUM_DOCS + " maxLength=" + maxLength + " allowDups=" + allowDups); } int numDocs = 0; IList <BytesRef> docValues = new List <BytesRef>(); // TODO: deletions while (numDocs < NUM_DOCS) { Document doc = new Document(); // 10% of the time, the document is missing the value: BytesRef br; if (LuceneTestCase.Random.Next(10) != 7) { string s; if (random.NextBoolean()) { s = TestUtil.RandomSimpleString(random, maxLength); } else { s = TestUtil.RandomUnicodeString(random, maxLength); } if (!allowDups) { if (seen.Contains(s)) { continue; } seen.Add(s); } if (Verbose) { Console.WriteLine(" " + numDocs + ": s=" + s); } br = new BytesRef(s); if (DefaultCodecSupportsDocValues) { doc.Add(new SortedDocValuesField("stringdv", br)); doc.Add(new NumericDocValuesField("id", numDocs)); } else { doc.Add(NewStringField("id", Convert.ToString(numDocs), Field.Store.NO)); } doc.Add(NewStringField("string", s, Field.Store.NO)); docValues.Add(br); } else { br = null; if (Verbose) { Console.WriteLine(" " + numDocs + ": <missing>"); } docValues.Add(null); if (DefaultCodecSupportsDocValues) { doc.Add(new NumericDocValuesField("id", numDocs)); } else { doc.Add(NewStringField("id", Convert.ToString(numDocs), Field.Store.NO)); } } doc.Add(new StoredField("id", numDocs)); writer.AddDocument(doc); numDocs++; if (random.Next(40) == 17) { // force flush writer.GetReader().Dispose(); } } IndexReader r = writer.GetReader(); writer.Dispose(); if (Verbose) { Console.WriteLine(" reader=" + r); } IndexSearcher idxS = NewSearcher( #if FEATURE_INSTANCE_TESTDATA_INITIALIZATION this, #endif r, false); int ITERS = AtLeast(100); for (int iter = 0; iter < ITERS; iter++) { bool reverse = random.NextBoolean(); TopFieldDocs hits; SortField sf; bool sortMissingLast; bool missingIsNull; if (DefaultCodecSupportsDocValues && random.NextBoolean()) { sf = new SortField("stringdv", SortFieldType.STRING, reverse); // Can only use sort missing if the DVFormat // supports docsWithField: sortMissingLast = DefaultCodecSupportsDocsWithField && Random.NextBoolean(); missingIsNull = DefaultCodecSupportsDocsWithField; } else { sf = new SortField("string", SortFieldType.STRING, reverse); sortMissingLast = Random.NextBoolean(); missingIsNull = true; } if (sortMissingLast) { sf.MissingValue = SortField.STRING_LAST; } Sort sort; if (random.NextBoolean()) { sort = new Sort(sf); } else { sort = new Sort(sf, SortField.FIELD_DOC); } int hitCount = TestUtil.NextInt32(random, 1, r.MaxDoc + 20); RandomFilter f = new RandomFilter(random, (float)random.NextDouble(), docValues); int queryType = random.Next(3); if (queryType == 0) { // force out of order BooleanQuery bq = new BooleanQuery(); // Add a Query with SHOULD, since bw.Scorer() returns BooleanScorer2 // which delegates to BS if there are no mandatory clauses. bq.Add(new MatchAllDocsQuery(), Occur.SHOULD); // Set minNrShouldMatch to 1 so that BQ will not optimize rewrite to return // the clause instead of BQ. bq.MinimumNumberShouldMatch = 1; hits = idxS.Search(bq, f, hitCount, sort, random.NextBoolean(), random.NextBoolean()); } else if (queryType == 1) { hits = idxS.Search(new ConstantScoreQuery(f), null, hitCount, sort, random.NextBoolean(), random.NextBoolean()); } else { hits = idxS.Search(new MatchAllDocsQuery(), f, hitCount, sort, random.NextBoolean(), random.NextBoolean()); } if (Verbose) { Console.WriteLine("\nTEST: iter=" + iter + " " + hits.TotalHits + " hits; topN=" + hitCount + "; reverse=" + reverse + "; sortMissingLast=" + sortMissingLast + " sort=" + sort); } // Compute expected results: var expected = f.matchValues.ToList(); expected.Sort(Comparer <BytesRef> .Create((a, b) => { if (a == null) { if (b == null) { return(0); } if (sortMissingLast) { return(1); } else { return(-1); } } else if (b == null) { if (sortMissingLast) { return(-1); } else { return(1); } } else { return(a.CompareTo(b)); } })); if (reverse) { expected.Reverse(); } if (Verbose) { Console.WriteLine(" expected:"); for (int idx = 0; idx < expected.Count; idx++) { BytesRef br = expected[idx]; if (br == null && missingIsNull == false) { br = new BytesRef(); } Console.WriteLine(" " + idx + ": " + (br == null ? "<missing>" : br.Utf8ToString())); if (idx == hitCount - 1) { break; } } } if (Verbose) { Console.WriteLine(" actual:"); for (int hitIDX = 0; hitIDX < hits.ScoreDocs.Length; hitIDX++) { FieldDoc fd = (FieldDoc)hits.ScoreDocs[hitIDX]; BytesRef br = (BytesRef)fd.Fields[0]; Console.WriteLine(" " + hitIDX + ": " + (br == null ? "<missing>" : br.Utf8ToString()) + " id=" + idxS.Doc(fd.Doc).Get("id")); } } for (int hitIDX = 0; hitIDX < hits.ScoreDocs.Length; hitIDX++) { FieldDoc fd = (FieldDoc)hits.ScoreDocs[hitIDX]; BytesRef br = expected[hitIDX]; if (br == null && missingIsNull == false) { br = new BytesRef(); } // Normally, the old codecs (that don't support // docsWithField via doc values) will always return // an empty BytesRef for the missing case; however, // if all docs in a given segment were missing, in // that case it will return null! So we must map // null here, too: BytesRef br2 = (BytesRef)fd.Fields[0]; if (br2 == null && missingIsNull == false) { br2 = new BytesRef(); } Assert.AreEqual(br, br2, "hit=" + hitIDX + " has wrong sort value"); } } r.Dispose(); dir.Dispose(); }
public override void Run() { try { while (Operations > 0) { int oper = rand.Next(100); if (oper < CommitPercent) { if (NumCommitting.IncrementAndGet() <= MaxConcurrentCommits) { IDictionary <int, long> newCommittedModel; long version; DirectoryReader oldReader; lock (OuterInstance) { newCommittedModel = new Dictionary <int, long>(OuterInstance.Model); // take a snapshot version = OuterInstance.SnapshotCount++; oldReader = OuterInstance.Reader; oldReader.IncRef(); // increment the reference since we will use this for reopening } DirectoryReader newReader; if (rand.Next(100) < SoftCommitPercent) { // assertU(h.Commit("softCommit","true")); if (Random.NextBoolean()) { if (VERBOSE) { Console.WriteLine("TEST: " + Thread.CurrentThread.Name + ": call writer.getReader"); } newReader = Writer.GetReader(true); } else { if (VERBOSE) { Console.WriteLine("TEST: " + Thread.CurrentThread.Name + ": reopen reader=" + oldReader + " version=" + version); } newReader = DirectoryReader.OpenIfChanged(oldReader, Writer.IndexWriter, true); } } else { // assertU(commit()); if (VERBOSE) { Console.WriteLine("TEST: " + Thread.CurrentThread.Name + ": commit+reopen reader=" + oldReader + " version=" + version); } Writer.Commit(); if (VERBOSE) { Console.WriteLine("TEST: " + Thread.CurrentThread.Name + ": now reopen after commit"); } newReader = DirectoryReader.OpenIfChanged(oldReader); } // Code below assumes newReader comes w/ // extra ref: if (newReader == null) { oldReader.IncRef(); newReader = oldReader; } oldReader.DecRef(); lock (OuterInstance) { // install the new reader if it's newest (and check the current version since another reader may have already been installed) //System.out.println(Thread.currentThread().getName() + ": newVersion=" + newReader.getVersion()); Debug.Assert(newReader.RefCount > 0); Debug.Assert(OuterInstance.Reader.RefCount > 0); if (newReader.Version > OuterInstance.Reader.Version) { if (VERBOSE) { Console.WriteLine("TEST: " + Thread.CurrentThread.Name + ": install new reader=" + newReader); } OuterInstance.Reader.DecRef(); OuterInstance.Reader = newReader; // Silly: forces fieldInfos to be // loaded so we don't hit IOE on later // reader.toString newReader.ToString(); // install this snapshot only if it's newer than the current one if (version >= OuterInstance.CommittedModelClock) { if (VERBOSE) { Console.WriteLine("TEST: " + Thread.CurrentThread.Name + ": install new model version=" + version); } OuterInstance.CommittedModel = newCommittedModel; OuterInstance.CommittedModelClock = version; } else { if (VERBOSE) { Console.WriteLine("TEST: " + Thread.CurrentThread.Name + ": skip install new model version=" + version); } } } else { // if the same reader, don't decRef. if (VERBOSE) { Console.WriteLine("TEST: " + Thread.CurrentThread.Name + ": skip install new reader=" + newReader); } newReader.DecRef(); } } } NumCommitting.DecrementAndGet(); } else { int id = rand.Next(Ndocs); object sync = OuterInstance.SyncArr[id]; // set the lastId before we actually change it sometimes to try and // uncover more race conditions between writing and reading bool before = Random.NextBoolean(); if (before) { OuterInstance.LastId = id; } // We can't concurrently update the same document and retain our invariants of increasing values // since we can't guarantee what order the updates will be executed. lock (sync) { long val = OuterInstance.Model[id]; long nextVal = Math.Abs(val) + 1; if (oper < CommitPercent + DeletePercent) { // assertU("<delete><id>" + id + "</id></delete>"); // add tombstone first if (Tombstones) { Document d = new Document(); d.Add(NewStringField("id", "-" + Convert.ToString(id), Documents.Field.Store.YES)); d.Add(NewField(OuterInstance.Field, Convert.ToString(nextVal), StoredOnlyType)); Writer.UpdateDocument(new Term("id", "-" + Convert.ToString(id)), d); } if (VERBOSE) { Console.WriteLine("TEST: " + Thread.CurrentThread.Name + ": term delDocs id:" + id + " nextVal=" + nextVal); } Writer.DeleteDocuments(new Term("id", Convert.ToString(id))); OuterInstance.Model[id] = -nextVal; } else if (oper < CommitPercent + DeletePercent + DeleteByQueryPercent) { //assertU("<delete><query>id:" + id + "</query></delete>"); // add tombstone first if (Tombstones) { Document d = new Document(); d.Add(NewStringField("id", "-" + Convert.ToString(id), Documents.Field.Store.YES)); d.Add(NewField(OuterInstance.Field, Convert.ToString(nextVal), StoredOnlyType)); Writer.UpdateDocument(new Term("id", "-" + Convert.ToString(id)), d); } if (VERBOSE) { Console.WriteLine("TEST: " + Thread.CurrentThread.Name + ": query delDocs id:" + id + " nextVal=" + nextVal); } Writer.DeleteDocuments(new TermQuery(new Term("id", Convert.ToString(id)))); OuterInstance.Model[id] = -nextVal; } else { // assertU(adoc("id",Integer.toString(id), field, Long.toString(nextVal))); Document d = new Document(); d.Add(NewStringField("id", Convert.ToString(id), Documents.Field.Store.YES)); d.Add(NewField(OuterInstance.Field, Convert.ToString(nextVal), StoredOnlyType)); if (VERBOSE) { Console.WriteLine("TEST: " + Thread.CurrentThread.Name + ": u id:" + id + " val=" + nextVal); } Writer.UpdateDocument(new Term("id", Convert.ToString(id)), d); if (Tombstones) { // remove tombstone after new addition (this should be optional?) Writer.DeleteDocuments(new Term("id", "-" + Convert.ToString(id))); } OuterInstance.Model[id] = nextVal; } } if (!before) { OuterInstance.LastId = id; } } } } catch (Exception e) { Console.WriteLine(Thread.CurrentThread.Name + ": FAILED: unexpected exception"); Console.WriteLine(e.StackTrace); throw new Exception(e.Message, e); } }
public override void Run() { try { IDictionary <string, int?> values = new Dictionary <string, int?>(); IList <string> allIDs = new SynchronizedList <string>(); startingGun.Wait(); for (int iter = 0; iter < iters; iter++) { // Add/update a document Document doc = new Document(); // Threads must not update the same id at the // same time: if (threadRandom.NextDouble() <= addChance) { string id = string.Format(CultureInfo.InvariantCulture, "{0}_{1:X4}", threadID, threadRandom.Next(idCount)); int field = threadRandom.Next(int.MaxValue); doc.Add(new StringField("id", id, Field.Store.YES)); doc.Add(new Int32Field("field", (int)field, Field.Store.YES)); w.UpdateDocument(new Term("id", id), doc); rt.Add(id, field); if (!values.ContainsKey(id))//Key didn't exist before { allIDs.Add(id); } values[id] = field; } if (allIDs.Count > 0 && threadRandom.NextDouble() <= deleteChance) { string randomID = allIDs[threadRandom.Next(allIDs.Count)]; w.DeleteDocuments(new Term("id", randomID)); rt.Delete(randomID); values[randomID] = missing; } if (threadRandom.NextDouble() <= reopenChance || rt.Count > 10000) { //System.out.println("refresh @ " + rt.Size()); mgr.MaybeRefresh(); if (Verbose) { IndexSearcher s = mgr.Acquire(); try { Console.WriteLine("TEST: reopen " + s); } finally { mgr.Release(s); } Console.WriteLine("TEST: " + values.Count + " values"); } } if (threadRandom.Next(10) == 7) { Assert.AreEqual(null, rt.Get("foo")); } if (allIDs.Count > 0) { string randomID = allIDs[threadRandom.Next(allIDs.Count)]; int? expected = values[randomID]; if (expected == missing) { expected = null; } Assert.AreEqual(expected, rt.Get(randomID), "id=" + randomID); } } } catch (Exception t) when(t.IsThrowable()) { throw RuntimeException.Create(t); } }
public override void Run() { try { IndexReader lastReader = null; IndexSearcher lastSearcher = null; while (Operations.DecrementAndGet() >= 0) { // bias toward a recently changed doc int id = rand.Next(100) < 25 ? OuterInstance.LastId : rand.Next(Ndocs); // when indexing, we update the index, then the model // so when querying, we should first check the model, and then the index long val; DirectoryReader r; lock (OuterInstance) { val = OuterInstance.CommittedModel[id]; r = OuterInstance.Reader; r.IncRef(); } if (VERBOSE) { Console.WriteLine("TEST: " + Thread.CurrentThread.Name + ": s id=" + id + " val=" + val + " r=" + r.Version); } // sreq = req("wt","json", "q","id:"+Integer.toString(id), "omitHeader","true"); IndexSearcher searcher; if (r == lastReader) { // Just re-use lastSearcher, else // newSearcher may create too many thread // pools (ExecutorService): searcher = lastSearcher; } else { searcher = NewSearcher( #if FEATURE_INSTANCE_TESTDATA_INITIALIZATION OuterInstance, #endif r); lastReader = r; lastSearcher = searcher; } Query q = new TermQuery(new Term("id", Convert.ToString(id))); TopDocs results = searcher.Search(q, 10); if (results.TotalHits == 0 && Tombstones) { // if we couldn't find the doc, look for its tombstone q = new TermQuery(new Term("id", "-" + Convert.ToString(id))); results = searcher.Search(q, 1); if (results.TotalHits == 0) { if (val == -1L) { // expected... no doc was added yet r.DecRef(); continue; } Assert.Fail("No documents or tombstones found for id " + id + ", expected at least " + val + " reader=" + r); } } if (results.TotalHits == 0 && !Tombstones) { // nothing to do - we can't tell anything from a deleted doc without tombstones } else { // we should have found the document, or its tombstone if (results.TotalHits != 1) { Console.WriteLine("FAIL: hits id:" + id + " val=" + val); foreach (ScoreDoc sd in results.ScoreDocs) { Document doc = r.Document(sd.Doc); Console.WriteLine(" docID=" + sd.Doc + " id:" + doc.Get("id") + " foundVal=" + doc.Get(OuterInstance.Field)); } Assert.Fail("id=" + id + " reader=" + r + " totalHits=" + results.TotalHits); } Document doc_ = searcher.Doc(results.ScoreDocs[0].Doc); long foundVal = Convert.ToInt64(doc_.Get(OuterInstance.Field)); if (foundVal < Math.Abs(val)) { Assert.Fail("foundVal=" + foundVal + " val=" + val + " id=" + id + " reader=" + r); } } r.DecRef(); } } catch (Exception e) { Operations.Value = ((int)-1L); Console.WriteLine(Thread.CurrentThread.Name + ": FAILED: unexpected exception"); Console.WriteLine(e.StackTrace); throw new Exception(e.Message, e); } }
public virtual void TestPayloadsPos0() { Directory dir = NewDirectory(); RandomIndexWriter writer = new RandomIndexWriter( #if FEATURE_INSTANCE_TESTDATA_INITIALIZATION this, #endif Random, dir, new MockPayloadAnalyzer()); Document doc = new Document(); doc.Add(new TextField("content", new StringReader("a a b c d e a f g h i j a b k k"))); writer.AddDocument(doc); IndexReader readerFromWriter = writer.GetReader(); AtomicReader r = SlowCompositeReaderWrapper.Wrap(readerFromWriter); DocsAndPositionsEnum tp = r.GetTermPositionsEnum(new Term("content", "a")); int count = 0; Assert.IsTrue(tp.NextDoc() != DocIdSetIterator.NO_MORE_DOCS); // "a" occurs 4 times Assert.AreEqual(4, tp.Freq); Assert.AreEqual(0, tp.NextPosition()); Assert.AreEqual(1, tp.NextPosition()); Assert.AreEqual(3, tp.NextPosition()); Assert.AreEqual(6, tp.NextPosition()); // only one doc has "a" Assert.AreEqual(DocIdSetIterator.NO_MORE_DOCS, tp.NextDoc()); IndexSearcher @is = NewSearcher(readerFromWriter); SpanTermQuery stq1 = new SpanTermQuery(new Term("content", "a")); SpanTermQuery stq2 = new SpanTermQuery(new Term("content", "k")); SpanQuery[] sqs = new SpanQuery[] { stq1, stq2 }; SpanNearQuery snq = new SpanNearQuery(sqs, 30, false); count = 0; bool sawZero = false; if (Verbose) { Console.WriteLine("\ngetPayloadSpans test"); } Search.Spans.Spans pspans = MultiSpansWrapper.Wrap(@is.TopReaderContext, snq); while (pspans.Next()) { if (Verbose) { Console.WriteLine("doc " + pspans.Doc + ": span " + pspans.Start + " to " + pspans.End); } var payloads = pspans.GetPayload(); sawZero |= pspans.Start == 0; foreach (var bytes in payloads) { count++; if (Verbose) { Console.WriteLine(" payload: " + Encoding.UTF8.GetString(bytes)); } } } Assert.IsTrue(sawZero); Assert.AreEqual(5, count); // System.out.println("\ngetSpans test"); Search.Spans.Spans spans = MultiSpansWrapper.Wrap(@is.TopReaderContext, snq); count = 0; sawZero = false; while (spans.Next()) { count++; sawZero |= spans.Start == 0; // System.out.println(spans.Doc() + " - " + spans.Start() + " - " + // spans.End()); } Assert.AreEqual(4, count); Assert.IsTrue(sawZero); // System.out.println("\nPayloadSpanUtil test"); sawZero = false; PayloadSpanUtil psu = new PayloadSpanUtil(@is.TopReaderContext); var pls = psu.GetPayloadsForQuery(snq); count = pls.Count; foreach (var bytes in pls) { string s = Encoding.UTF8.GetString(bytes); //System.out.println(s); sawZero |= s.Equals("pos: 0", StringComparison.Ordinal); } Assert.AreEqual(5, count); Assert.IsTrue(sawZero); writer.Dispose(); @is.IndexReader.Dispose(); dir.Dispose(); }
public virtual void Test() { // update variables int commitPercent = Random.Next(20); int softCommitPercent = Random.Next(100); // what percent of the commits are soft int deletePercent = Random.Next(50); int deleteByQueryPercent = Random.Next(25); int ndocs = AtLeast(50); int nWriteThreads = TestUtil.NextInt32(Random, 1, TEST_NIGHTLY ? 10 : 5); int maxConcurrentCommits = TestUtil.NextInt32(Random, 1, TEST_NIGHTLY ? 10 : 5); // number of committers at a time... needed if we want to avoid commit errors due to exceeding the max bool tombstones = Random.NextBoolean(); // query variables AtomicInt64 operations = new AtomicInt64(AtLeast(10000)); // number of query operations to perform in total int nReadThreads = TestUtil.NextInt32(Random, 1, TEST_NIGHTLY ? 10 : 5); InitModel(ndocs); FieldType storedOnlyType = new FieldType(); storedOnlyType.IsStored = true; if (VERBOSE) { Console.WriteLine("\n"); Console.WriteLine("TEST: commitPercent=" + commitPercent); Console.WriteLine("TEST: softCommitPercent=" + softCommitPercent); Console.WriteLine("TEST: deletePercent=" + deletePercent); Console.WriteLine("TEST: deleteByQueryPercent=" + deleteByQueryPercent); Console.WriteLine("TEST: ndocs=" + ndocs); Console.WriteLine("TEST: nWriteThreads=" + nWriteThreads); Console.WriteLine("TEST: nReadThreads=" + nReadThreads); Console.WriteLine("TEST: maxConcurrentCommits=" + maxConcurrentCommits); Console.WriteLine("TEST: tombstones=" + tombstones); Console.WriteLine("TEST: operations=" + operations); Console.WriteLine("\n"); } AtomicInt32 numCommitting = new AtomicInt32(); IList <ThreadJob> threads = new List <ThreadJob>(); Directory dir = NewDirectory(); RandomIndexWriter writer = new RandomIndexWriter(Random, dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random))); writer.DoRandomForceMergeAssert = false; writer.Commit(); Reader = DirectoryReader.Open(dir); for (int i = 0; i < nWriteThreads; i++) { ThreadJob thread = new ThreadAnonymousInnerClassHelper(this, "WRITER" + i, commitPercent, softCommitPercent, deletePercent, deleteByQueryPercent, ndocs, maxConcurrentCommits, tombstones, operations, storedOnlyType, numCommitting, writer); threads.Add(thread); } for (int i = 0; i < nReadThreads; i++) { ThreadJob thread = new ThreadAnonymousInnerClassHelper2(this, "READER" + i, ndocs, tombstones, operations); threads.Add(thread); } foreach (ThreadJob thread in threads) { thread.Start(); } foreach (ThreadJob thread in threads) { thread.Join(); } writer.Dispose(); if (VERBOSE) { Console.WriteLine("TEST: close reader=" + Reader); } Reader.Dispose(); dir.Dispose(); }
public virtual void TestCommitOnCloseForceMerge() { Directory dir = NewDirectory(); // Must disable throwing exc on double-write: this // test uses IW.rollback which easily results in // writing to same file more than once if (dir is MockDirectoryWrapper) { ((MockDirectoryWrapper)dir).PreventDoubleWrite = false; } IndexWriter writer = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random)).SetMaxBufferedDocs(10).SetMergePolicy(NewLogMergePolicy(10))); for (int j = 0; j < 17; j++) { AddDocWithIndex(writer, j); } writer.Dispose(); writer = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random)).SetOpenMode(OpenMode.APPEND)); writer.ForceMerge(1); // Open a reader before closing (commiting) the writer: DirectoryReader reader = DirectoryReader.Open(dir); // Reader should see index as multi-seg at this // point: Assert.IsTrue(reader.Leaves.Count > 1, "Reader incorrectly sees one segment"); reader.Dispose(); // Abort the writer: writer.Rollback(); TestIndexWriter.AssertNoUnreferencedFiles(dir, "aborted writer after forceMerge"); // Open a reader after aborting writer: reader = DirectoryReader.Open(dir); // Reader should still see index as multi-segment Assert.IsTrue(reader.Leaves.Count > 1, "Reader incorrectly sees one segment"); reader.Dispose(); if (VERBOSE) { Console.WriteLine("TEST: do real full merge"); } writer = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random)).SetOpenMode(OpenMode.APPEND)); writer.ForceMerge(1); writer.Dispose(); if (VERBOSE) { Console.WriteLine("TEST: writer closed"); } TestIndexWriter.AssertNoUnreferencedFiles(dir, "aborted writer after forceMerge"); // Open a reader after aborting writer: reader = DirectoryReader.Open(dir); // Reader should see index as one segment Assert.AreEqual(1, reader.Leaves.Count, "Reader incorrectly sees more than one segment"); reader.Dispose(); dir.Dispose(); }
private PreviousSearchState AssertSame(IndexSearcher mockSearcher, NodeState.ShardIndexSearcher shardSearcher, Query q, Sort sort, PreviousSearchState state) { int numHits = TestUtil.NextInt32(Random, 1, 100); if (state != null && state.SearchAfterLocal == null) { // In addition to what we last searched: numHits += state.NumHitsPaged; } if (VERBOSE) { Console.WriteLine("TEST: query=" + q + " sort=" + sort + " numHits=" + numHits); if (state != null) { Console.WriteLine(" prev: searchAfterLocal=" + state.SearchAfterLocal + " searchAfterShard=" + state.SearchAfterShard + " numHitsPaged=" + state.NumHitsPaged); } } // Single (mock local) searcher: TopDocs hits; if (sort == null) { if (state != null && state.SearchAfterLocal != null) { hits = mockSearcher.SearchAfter(state.SearchAfterLocal, q, numHits); } else { hits = mockSearcher.Search(q, numHits); } } else { hits = mockSearcher.Search(q, numHits, sort); } // Shard searcher TopDocs shardHits; if (sort == null) { if (state != null && state.SearchAfterShard != null) { shardHits = shardSearcher.SearchAfter(state.SearchAfterShard, q, numHits); } else { shardHits = shardSearcher.Search(q, numHits); } } else { shardHits = shardSearcher.Search(q, numHits, sort); } int numNodes = shardSearcher.GetNodeVersions().Length; int[] @base = new int[numNodes]; IList <IndexReaderContext> subs = mockSearcher.TopReaderContext.Children; Assert.AreEqual(numNodes, subs.Count); for (int nodeID = 0; nodeID < numNodes; nodeID++) { @base[nodeID] = subs[nodeID].DocBaseInParent; } if (VERBOSE) { /* * for(int shardID=0;shardID<shardSearchers.Length;shardID++) { * System.out.println(" shard=" + shardID + " maxDoc=" + shardSearchers[shardID].searcher.getIndexReader().MaxDoc); * } */ Console.WriteLine(" single searcher: " + hits.TotalHits + " totalHits maxScore=" + hits.MaxScore); for (int i = 0; i < hits.ScoreDocs.Length; i++) { ScoreDoc sd = hits.ScoreDocs[i]; Console.WriteLine(" doc=" + sd.Doc + " score=" + sd.Score); } Console.WriteLine(" shard searcher: " + shardHits.TotalHits + " totalHits maxScore=" + shardHits.MaxScore); for (int i = 0; i < shardHits.ScoreDocs.Length; i++) { ScoreDoc sd = shardHits.ScoreDocs[i]; Console.WriteLine(" doc=" + sd.Doc + " (rebased: " + (sd.Doc + @base[sd.ShardIndex]) + ") score=" + sd.Score + " shard=" + sd.ShardIndex); } } int numHitsPaged; if (state != null && state.SearchAfterLocal != null) { numHitsPaged = hits.ScoreDocs.Length; if (state != null) { numHitsPaged += state.NumHitsPaged; } } else { numHitsPaged = hits.ScoreDocs.Length; } bool moreHits; ScoreDoc bottomHit; ScoreDoc bottomHitShards; if (numHitsPaged < hits.TotalHits) { // More hits to page through moreHits = true; if (sort == null) { bottomHit = hits.ScoreDocs[hits.ScoreDocs.Length - 1]; ScoreDoc sd = shardHits.ScoreDocs[shardHits.ScoreDocs.Length - 1]; // Must copy because below we rebase: bottomHitShards = new ScoreDoc(sd.Doc, sd.Score, sd.ShardIndex); if (VERBOSE) { Console.WriteLine(" save bottomHit=" + bottomHit); } } else { bottomHit = null; bottomHitShards = null; } } else { Assert.AreEqual(hits.TotalHits, numHitsPaged); bottomHit = null; bottomHitShards = null; moreHits = false; } // Must rebase so Assert.AreEqual passes: for (int hitID = 0; hitID < shardHits.ScoreDocs.Length; hitID++) { ScoreDoc sd = shardHits.ScoreDocs[hitID]; sd.Doc += @base[sd.ShardIndex]; } TestUtil.AssertEquals(hits, shardHits); if (moreHits) { // Return a continuation: return(new PreviousSearchState(q, sort, bottomHit, bottomHitShards, shardSearcher.GetNodeVersions(), numHitsPaged)); } else { return(null); } }