public abstract GetSortedSetDocValues ( string field ) : Lucene.Net.Index.SortedSetDocValues | ||
field | string | |
return | Lucene.Net.Index.SortedSetDocValues |
public override void Warm(AtomicReader reader) { long startTime = Environment.TickCount; int indexedCount = 0; int docValuesCount = 0; int normsCount = 0; foreach (FieldInfo info in reader.FieldInfos) { if (info.IsIndexed) { reader.GetTerms(info.Name); indexedCount++; if (info.HasNorms) { reader.GetNormValues(info.Name); normsCount++; } } if (info.HasDocValues) { switch (info.DocValuesType) { case DocValuesType.NUMERIC: reader.GetNumericDocValues(info.Name); break; case DocValuesType.BINARY: reader.GetBinaryDocValues(info.Name); break; case DocValuesType.SORTED: reader.GetSortedDocValues(info.Name); break; case DocValuesType.SORTED_SET: reader.GetSortedSetDocValues(info.Name); break; default: if (Debugging.AssertsEnabled) { Debugging.Assert(false); // unknown dv type } break; } docValuesCount++; } } reader.Document(0); reader.GetTermVectors(0); if (infoStream.IsEnabled("SMSW")) { infoStream.Message("SMSW", "Finished warming segment: " + reader + ", indexed=" + indexedCount + ", docValues=" + docValuesCount + ", norms=" + normsCount + ", time=" + (Environment.TickCount - startTime)); } }
public override void Warm(AtomicReader reader) { long startTime = DateTime.Now.Millisecond; int indexedCount = 0; int docValuesCount = 0; int normsCount = 0; foreach (FieldInfo info in reader.FieldInfos) { if (info.Indexed) { reader.Terms(info.Name); indexedCount++; if (info.HasNorms()) { reader.GetNormValues(info.Name); normsCount++; } } if (info.HasDocValues()) { switch (info.DocValuesType) { case DocValuesType_e.NUMERIC: reader.GetNumericDocValues(info.Name); break; case DocValuesType_e.BINARY: reader.GetBinaryDocValues(info.Name); break; case DocValuesType_e.SORTED: reader.GetSortedDocValues(info.Name); break; case DocValuesType_e.SORTED_SET: reader.GetSortedSetDocValues(info.Name); break; default: Debug.Assert(false); // unknown dv type break; } docValuesCount++; } } reader.Document(0); reader.GetTermVectors(0); if (InfoStream.IsEnabled("SMSW")) { InfoStream.Message("SMSW", "Finished warming segment: " + reader + ", indexed=" + indexedCount + ", docValues=" + docValuesCount + ", norms=" + normsCount + ", time=" + (DateTime.Now.Millisecond - startTime)); } }
/// <summary> /// Creates this, pulling doc values from the specified /// field. /// </summary> public DefaultSortedSetDocValuesReaderState(IndexReader reader, string field = FacetsConfig.DEFAULT_INDEX_FIELD_NAME) { this.field = field; this.origReader = reader; // We need this to create thread-safe MultiSortedSetDV // per collector: topReader = SlowCompositeReaderWrapper.Wrap(reader); SortedSetDocValues dv = topReader.GetSortedSetDocValues(field); if (dv == null) { throw new System.ArgumentException("field \"" + field + "\" was not indexed with SortedSetDocValues"); } if (dv.ValueCount > int.MaxValue) { throw new System.ArgumentException("can only handle valueCount < Integer.MAX_VALUE; got " + dv.ValueCount); } valueCount = (int)dv.ValueCount; // TODO: we can make this more efficient if eg we can be // "involved" when IOrdinalMap is being created? Ie see // each term/ord it's assigning as it goes... string lastDim = null; int startOrd = -1; // TODO: this approach can work for full hierarchy?; // TaxoReader can't do this since ords are not in // "sorted order" ... but we should generalize this to // support arbitrary hierarchy: for (int ord = 0; ord < valueCount; ord++) { BytesRef term = new BytesRef(); dv.LookupOrd(ord, term); string[] components = FacetsConfig.StringToPath(term.Utf8ToString()); if (components.Length != 2) { throw new System.ArgumentException("this class can only handle 2 level hierarchy (dim/value); got: " + Arrays.ToString(components) + " " + term.Utf8ToString()); } if (!components[0].Equals(lastDim)) { if (lastDim != null) { prefixToOrdRange[lastDim] = new OrdRange(startOrd, ord - 1); } startOrd = ord; lastDim = components[0]; } } if (lastDim != null) { prefixToOrdRange[lastDim] = new OrdRange(startOrd, valueCount - 1); } }
public virtual void TestSortedSetWithDups() { AssumeTrue("codec does not support SORTED_SET", DefaultCodecSupportsSortedSet()); Directory dir = NewDirectory(); IndexWriterConfig iwc = NewIndexWriterConfig(Random(), TEST_VERSION_CURRENT, null); iwc.SetMergePolicy(NewLogMergePolicy()); RandomIndexWriter iw = new RandomIndexWriter(Random(), dir, iwc); int numDocs = AtLeast(500); for (int i = 0; i < numDocs; i++) { Document doc = new Document(); int numValues = Random().Next(5); for (int j = 0; j < numValues; j++) { doc.Add(new SortedSetDocValuesField("bytes", new BytesRef(TestUtil.RandomSimpleString(Random(), 2)))); } iw.AddDocument(doc); if (Random().Next(17) == 0) { iw.Commit(); } } DirectoryReader ir = iw.Reader; iw.ForceMerge(1); DirectoryReader ir2 = iw.Reader; AtomicReader merged = GetOnlySegmentReader(ir2); iw.Dispose(); SortedSetDocValues multi = MultiDocValues.GetSortedSetValues(ir, "bytes"); SortedSetDocValues single = merged.GetSortedSetDocValues("bytes"); if (multi == null) { Assert.IsNull(single); } else { Assert.AreEqual(single.ValueCount, multi.ValueCount); BytesRef actual = new BytesRef(); BytesRef expected = new BytesRef(); // check values for (long i = 0; i < single.ValueCount; i++) { single.LookupOrd(i, expected); multi.LookupOrd(i, actual); Assert.AreEqual(expected, actual); } // check ord list for (int i = 0; i < numDocs; i++) { single.SetDocument(i); List <long?> expectedList = new List <long?>(); long ord; while ((ord = single.NextOrd()) != SortedSetDocValues.NO_MORE_ORDS) { expectedList.Add(ord); } multi.SetDocument(i); int upto = 0; while ((ord = multi.NextOrd()) != SortedSetDocValues.NO_MORE_ORDS) { Assert.AreEqual((long)expectedList[upto], ord); upto++; } Assert.AreEqual(expectedList.Count, upto); } } ir.Dispose(); ir2.Dispose(); dir.Dispose(); }
/// <summary> /// Test docvalues. /// @lucene.experimental /// </summary> public static Status.DocValuesStatus TestDocValues(AtomicReader reader, TextWriter infoStream) { Status.DocValuesStatus status = new Status.DocValuesStatus(); try { if (infoStream != null) { infoStream.Write(" test: docvalues..........."); } foreach (FieldInfo fieldInfo in reader.FieldInfos) { if (fieldInfo.HasDocValues()) { status.TotalValueFields++; CheckDocValues(fieldInfo, reader, /*infoStream,*/ status); } else { if (reader.GetBinaryDocValues(fieldInfo.Name) != null || reader.GetNumericDocValues(fieldInfo.Name) != null || reader.GetSortedDocValues(fieldInfo.Name) != null || reader.GetSortedSetDocValues(fieldInfo.Name) != null || reader.GetDocsWithField(fieldInfo.Name) != null) { throw new Exception("field: " + fieldInfo.Name + " has docvalues but should omit them!"); } } } Msg(infoStream, "OK [" + status.TotalValueFields + " docvalues fields; " + status.TotalBinaryFields + " BINARY; " + status.TotalNumericFields + " NUMERIC; " + status.TotalSortedFields + " SORTED; " + status.TotalSortedSetFields + " SORTED_SET]"); } catch (Exception e) { Msg(infoStream, "ERROR [" + Convert.ToString(e.Message) + "]"); status.Error = e; if (infoStream != null) { // LUCENENET NOTE: Some tests rely on the error type being in // the message. We can't get the error type with StackTrace, we // need ToString() for that. infoStream.WriteLine(e.ToString()); //infoStream.WriteLine(e.StackTrace); } } return status; }
private static void CheckDocValues(FieldInfo fi, AtomicReader reader, /*StreamWriter infoStream,*/ DocValuesStatus status) { Bits docsWithField = reader.GetDocsWithField(fi.Name); if (docsWithField == null) { throw new Exception(fi.Name + " docsWithField does not exist"); } else if (docsWithField.Length() != reader.MaxDoc) { throw new Exception(fi.Name + " docsWithField has incorrect length: " + docsWithField.Length() + ",expected: " + reader.MaxDoc); } switch (fi.DocValuesType) { case FieldInfo.DocValuesType_e.SORTED: status.TotalSortedFields++; CheckSortedDocValues(fi.Name, reader, reader.GetSortedDocValues(fi.Name), docsWithField); if (reader.GetBinaryDocValues(fi.Name) != null || reader.GetNumericDocValues(fi.Name) != null || reader.GetSortedSetDocValues(fi.Name) != null) { throw new Exception(fi.Name + " returns multiple docvalues types!"); } break; case FieldInfo.DocValuesType_e.SORTED_SET: status.TotalSortedSetFields++; CheckSortedSetDocValues(fi.Name, reader, reader.GetSortedSetDocValues(fi.Name), docsWithField); if (reader.GetBinaryDocValues(fi.Name) != null || reader.GetNumericDocValues(fi.Name) != null || reader.GetSortedDocValues(fi.Name) != null) { throw new Exception(fi.Name + " returns multiple docvalues types!"); } break; case FieldInfo.DocValuesType_e.BINARY: status.TotalBinaryFields++; CheckBinaryDocValues(fi.Name, reader, reader.GetBinaryDocValues(fi.Name), docsWithField); if (reader.GetNumericDocValues(fi.Name) != null || reader.GetSortedDocValues(fi.Name) != null || reader.GetSortedSetDocValues(fi.Name) != null) { throw new Exception(fi.Name + " returns multiple docvalues types!"); } break; case FieldInfo.DocValuesType_e.NUMERIC: status.TotalNumericFields++; CheckNumericDocValues(fi.Name, reader, reader.GetNumericDocValues(fi.Name), docsWithField); if (reader.GetBinaryDocValues(fi.Name) != null || reader.GetSortedDocValues(fi.Name) != null || reader.GetSortedSetDocValues(fi.Name) != null) { throw new Exception(fi.Name + " returns multiple docvalues types!"); } break; default: throw new InvalidOperationException(); } }
public override void Warm(AtomicReader reader) { long startTime = J2N.Time.NanoTime() / J2N.Time.MillisecondsPerNanosecond; // LUCENENET: Use NanoTime() rather than CurrentTimeMilliseconds() for more accurate/reliable results int indexedCount = 0; int docValuesCount = 0; int normsCount = 0; foreach (FieldInfo info in reader.FieldInfos) { if (info.IsIndexed) { reader.GetTerms(info.Name); indexedCount++; if (info.HasNorms) { reader.GetNormValues(info.Name); normsCount++; } } if (info.HasDocValues) { switch (info.DocValuesType) { case DocValuesType.NUMERIC: reader.GetNumericDocValues(info.Name); break; case DocValuesType.BINARY: reader.GetBinaryDocValues(info.Name); break; case DocValuesType.SORTED: reader.GetSortedDocValues(info.Name); break; case DocValuesType.SORTED_SET: reader.GetSortedSetDocValues(info.Name); break; default: if (Debugging.AssertsEnabled) { Debugging.Assert(false); // unknown dv type } break; } docValuesCount++; } } reader.Document(0); reader.GetTermVectors(0); if (infoStream.IsEnabled("SMSW")) { infoStream.Message("SMSW", "Finished warming segment: " + reader + ", indexed=" + indexedCount + ", docValues=" + docValuesCount + ", norms=" + normsCount + ", time=" + ((J2N.Time.NanoTime() / J2N.Time.MillisecondsPerNanosecond) - startTime)); // LUCENENET: Use NanoTime() rather than CurrentTimeMilliseconds() for more accurate/reliable results } }
public override SortedSetDocValues GetSortedSetDocValues(string field) { EnsureOpen(); return(m_input.GetSortedSetDocValues(field)); }
internal SlowMinShouldMatchScorer(BooleanWeight weight, AtomicReader reader, IndexSearcher searcher) : base(weight) { this.Dv = reader.GetSortedSetDocValues("dv"); this.MaxDoc = reader.MaxDoc; BooleanQuery bq = (BooleanQuery)weight.Query; this.MinNrShouldMatch = bq.MinimumNumberShouldMatch; this.Sims = new SimScorer[(int)Dv.ValueCount]; foreach (BooleanClause clause in bq.Clauses) { Debug.Assert(!clause.Prohibited); Debug.Assert(!clause.Required); Term term = ((TermQuery)clause.Query).Term; long ord = Dv.LookupTerm(term.Bytes); if (ord >= 0) { bool success = Ords.Add(ord); Debug.Assert(success); // no dups TermContext context = TermContext.Build(reader.Context, term); SimWeight w = weight.Similarity.ComputeWeight(1f, searcher.CollectionStatistics("field"), searcher.TermStatistics(term, context)); var dummy = w.ValueForNormalization; // ignored w.Normalize(1F, 1F); Sims[(int)ord] = weight.Similarity.DoSimScorer(w, (AtomicReaderContext)reader.Context); } } }