public virtual void TestTermOrd() { Directory d = NewDirectory(); IndexWriter w = new IndexWriter(d, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetCodec(TestUtil.AlwaysPostingsFormat(new Lucene41PostingsFormat()))); Documents.Document doc = new Documents.Document(); doc.Add(NewTextField("f", "a b c", Field.Store.NO)); w.AddDocument(doc); w.ForceMerge(1); DirectoryReader r = w.Reader; TermsEnum terms = GetOnlySegmentReader(r).Fields.Terms("f").Iterator(null); Assert.IsTrue(terms.Next() != null); try { Assert.AreEqual(0, terms.Ord()); } catch (System.NotSupportedException uoe) { // ok -- codec is not required to support this op } r.Dispose(); w.Dispose(); d.Dispose(); }
public virtual void TestPrevTermAtEnd() { IndexWriter writer = new IndexWriter(Dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetCodec(TestUtil.AlwaysPostingsFormat(new Lucene41PostingsFormat()))); AddDoc(writer, "aaa bbb"); writer.Dispose(); SegmentReader reader = GetOnlySegmentReader(DirectoryReader.Open(Dir)); TermsEnum terms = reader.Fields.Terms("content").Iterator(null); Assert.IsNotNull(terms.Next()); Assert.AreEqual("aaa", terms.Term().Utf8ToString()); Assert.IsNotNull(terms.Next()); long ordB; try { ordB = terms.Ord(); } catch (System.NotSupportedException uoe) { // ok -- codec is not required to support ord reader.Dispose(); return; } Assert.AreEqual("bbb", terms.Term().Utf8ToString()); Assert.IsNull(terms.Next()); terms.SeekExact(ordB); Assert.AreEqual("bbb", terms.Term().Utf8ToString()); reader.Dispose(); }
public override long LookupTerm(BytesRef key) { try { if (Te.SeekCeil(key) == SeekStatus.FOUND) { return(Te.Ord()); } else { return(-Te.Ord() - 1); } } catch (System.IO.IOException e) { throw new Exception(e.Message, e); } }
/// <summary> /// Call this only once (if you subclass!) </summary> protected internal virtual void Uninvert(AtomicReader reader, Bits liveDocs, BytesRef termPrefix) { FieldInfo info = reader.FieldInfos.FieldInfo(Field); if (info != null && info.HasDocValues()) { throw new InvalidOperationException("Type mismatch: " + Field + " was indexed as " + info.DocValuesType); } //System.out.println("DTO uninvert field=" + field + " prefix=" + termPrefix); long startTime = DateTime.Now.Millisecond; Prefix = termPrefix == null ? null : BytesRef.DeepCopyOf(termPrefix); int maxDoc = reader.MaxDoc; int[] index = new int[maxDoc]; // immediate term numbers, or the index into the byte[] representing the last number int[] lastTerm = new int[maxDoc]; // last term we saw for this document sbyte[][] bytes = new sbyte[maxDoc][]; // list of term numbers for the doc (delta encoded vInts) Fields fields = reader.Fields; if (fields == null) { // No terms return; } Terms terms = fields.Terms(Field); if (terms == null) { // No terms return; } TermsEnum te = terms.Iterator(null); BytesRef seekStart = termPrefix != null ? termPrefix : new BytesRef(); //System.out.println("seekStart=" + seekStart.utf8ToString()); if (te.SeekCeil(seekStart) == TermsEnum.SeekStatus.END) { // No terms match return; } // If we need our "term index wrapper", these will be // init'd below: IList <BytesRef> indexedTerms = null; PagedBytes indexedTermsBytes = null; bool testedOrd = false; // we need a minimum of 9 bytes, but round up to 12 since the space would // be wasted with most allocators anyway. sbyte[] tempArr = new sbyte[12]; // // enumerate all terms, and build an intermediate form of the un-inverted field. // // During this intermediate form, every document has a (potential) byte[] // and the int[maxDoc()] array either contains the termNumber list directly // or the *end* offset of the termNumber list in it's byte array (for faster // appending and faster creation of the final form). // // idea... if things are too large while building, we could do a range of docs // at a time (but it would be a fair amount slower to build) // could also do ranges in parallel to take advantage of multiple CPUs // OPTIONAL: remap the largest df terms to the lowest 128 (single byte) // values. this requires going over the field first to find the most // frequent terms ahead of time. int termNum = 0; DocsEnum = null; // Loop begins with te positioned to first term (we call // seek above): for (; ;) { BytesRef t = te.Term(); if (t == null || (termPrefix != null && !StringHelper.StartsWith(t, termPrefix))) { break; } //System.out.println("visit term=" + t.utf8ToString() + " " + t + " termNum=" + termNum); if (!testedOrd) { try { OrdBase = (int)te.Ord(); //System.out.println("got ordBase=" + ordBase); } catch (System.NotSupportedException uoe) { // Reader cannot provide ord support, so we wrap // our own support by creating our own terms index: indexedTerms = new List <BytesRef>(); indexedTermsBytes = new PagedBytes(15); //System.out.println("NO ORDS"); } testedOrd = true; } VisitTerm(te, termNum); if (indexedTerms != null && (termNum & IndexIntervalMask) == 0) { // Index this term SizeOfIndexedStrings += t.Length; BytesRef indexedTerm = new BytesRef(); indexedTermsBytes.Copy(t, indexedTerm); // TODO: really should 1) strip off useless suffix, // and 2) use FST not array/PagedBytes indexedTerms.Add(indexedTerm); } int df = te.DocFreq(); if (df <= MaxTermDocFreq) { DocsEnum = te.Docs(liveDocs, DocsEnum, DocsEnum.FLAG_NONE); // dF, but takes deletions into account int actualDF = 0; for (; ;) { int doc = DocsEnum.NextDoc(); if (doc == DocIdSetIterator.NO_MORE_DOCS) { break; } //System.out.println(" chunk=" + chunk + " docs"); actualDF++; TermInstances++; //System.out.println(" docID=" + doc); // add TNUM_OFFSET to the term number to make room for special reserved values: // 0 (end term) and 1 (index into byte array follows) int delta = termNum - lastTerm[doc] + TNUM_OFFSET; lastTerm[doc] = termNum; int val = index[doc]; if ((val & 0xff) == 1) { // index into byte array (actually the end of // the doc-specific byte[] when building) int pos = (int)((uint)val >> 8); int ilen = VIntSize(delta); sbyte[] arr = bytes[doc]; int newend = pos + ilen; if (newend > arr.Length) { // We avoid a doubling strategy to lower memory usage. // this faceting method isn't for docs with many terms. // In hotspot, objects have 2 words of overhead, then fields, rounded up to a 64-bit boundary. // TODO: figure out what array lengths we can round up to w/o actually using more memory // (how much space does a byte[] take up? Is data preceded by a 32 bit length only? // It should be safe to round up to the nearest 32 bits in any case. int newLen = (newend + 3) & unchecked ((int)0xfffffffc); // 4 byte alignment sbyte[] newarr = new sbyte[newLen]; Array.Copy(arr, 0, newarr, 0, pos); arr = newarr; bytes[doc] = newarr; } pos = WriteInt(delta, arr, pos); index[doc] = (pos << 8) | 1; // update pointer to end index in byte[] } else { // OK, this int has data in it... find the end (a zero starting byte - not // part of another number, hence not following a byte with the high bit set). int ipos; if (val == 0) { ipos = 0; } else if ((val & 0x0000ff80) == 0) { ipos = 1; } else if ((val & 0x00ff8000) == 0) { ipos = 2; } else if ((val & 0xff800000) == 0) { ipos = 3; } else { ipos = 4; } //System.out.println(" ipos=" + ipos); int endPos = WriteInt(delta, tempArr, ipos); //System.out.println(" endpos=" + endPos); if (endPos <= 4) { //System.out.println(" fits!"); // value will fit in the integer... move bytes back for (int j = ipos; j < endPos; j++) { val |= (tempArr[j] & 0xff) << (j << 3); } index[doc] = val; } else { // value won't fit... move integer into byte[] for (int j = 0; j < ipos; j++) { tempArr[j] = (sbyte)val; val = (int)((uint)val >> 8); } // point at the end index in the byte[] index[doc] = (endPos << 8) | 1; bytes[doc] = tempArr; tempArr = new sbyte[12]; } } } SetActualDocFreq(termNum, actualDF); } termNum++; if (te.Next() == null) { break; } } NumTermsInField = termNum; long midPoint = DateTime.Now.Millisecond; if (TermInstances == 0) { // we didn't invert anything // lower memory consumption. Tnums = null; } else { this.Index = index; // // transform intermediate form into the final form, building a single byte[] // at a time, and releasing the intermediate byte[]s as we go to avoid // increasing the memory footprint. // for (int pass = 0; pass < 256; pass++) { sbyte[] target = Tnums[pass]; int pos = 0; // end in target; if (target != null) { pos = target.Length; } else { target = new sbyte[4096]; } // loop over documents, 0x00ppxxxx, 0x01ppxxxx, 0x02ppxxxx // where pp is the pass (which array we are building), and xx is all values. // each pass shares the same byte[] for termNumber lists. for (int docbase = pass << 16; docbase < maxDoc; docbase += (1 << 24)) { int lim = Math.Min(docbase + (1 << 16), maxDoc); for (int doc = docbase; doc < lim; doc++) { //System.out.println(" pass="******" process docID=" + doc); int val = index[doc]; if ((val & 0xff) == 1) { int len = (int)((uint)val >> 8); //System.out.println(" ptr pos=" + pos); index[doc] = (pos << 8) | 1; // change index to point to start of array if ((pos & 0xff000000) != 0) { // we only have 24 bits for the array index throw new InvalidOperationException("Too many values for UnInvertedField faceting on field " + Field); } sbyte[] arr = bytes[doc]; /* * for(byte b : arr) { * //System.out.println(" b=" + Integer.toHexString((int) b)); * } */ bytes[doc] = null; // IMPORTANT: allow GC to avoid OOM if (target.Length <= pos + len) { int newlen = target.Length; /// <summary> ///* we don't have to worry about the array getting too large /// since the "pos" param will overflow first (only 24 bits available) /// if ((newlen<<1) <= 0) { /// // overflow... /// newlen = Integer.MAX_VALUE; /// if (newlen <= pos + len) { /// throw new SolrException(400,"Too many terms to uninvert field!"); /// } /// } else { /// while (newlen <= pos + len) newlen<<=1; // doubling strategy /// } /// *** /// </summary> while (newlen <= pos + len) // doubling strategy { newlen <<= 1; } sbyte[] newtarget = new sbyte[newlen]; Array.Copy(target, 0, newtarget, 0, pos); target = newtarget; } Array.Copy(arr, 0, target, pos, len); pos += len + 1; // skip single byte at end and leave it 0 for terminator } } } // shrink array if (pos < target.Length) { sbyte[] newtarget = new sbyte[pos]; Array.Copy(target, 0, newtarget, 0, pos); target = newtarget; } Tnums[pass] = target; if ((pass << 16) > maxDoc) { break; } } } if (indexedTerms != null) { IndexedTermsArray = indexedTerms.ToArray(); } long endTime = DateTime.Now.Millisecond; Total_time = (int)(endTime - startTime); Phase1_time = (int)(midPoint - startTime); }
public override long Ord() { return(Tenum.Ord()); }
public virtual void TestSortedTermsEnum() { Directory directory = NewDirectory(); Analyzer analyzer = new MockAnalyzer(Random()); IndexWriterConfig iwconfig = NewIndexWriterConfig(TEST_VERSION_CURRENT, analyzer); iwconfig.SetMergePolicy(NewLogMergePolicy()); RandomIndexWriter iwriter = new RandomIndexWriter(Random(), directory, iwconfig); Document doc = new Document(); doc.Add(new StringField("field", "hello", Field.Store.NO)); iwriter.AddDocument(doc); doc = new Document(); doc.Add(new StringField("field", "world", Field.Store.NO)); iwriter.AddDocument(doc); doc = new Document(); doc.Add(new StringField("field", "beer", Field.Store.NO)); iwriter.AddDocument(doc); iwriter.ForceMerge(1); DirectoryReader ireader = iwriter.Reader; iwriter.Dispose(); AtomicReader ar = GetOnlySegmentReader(ireader); SortedSetDocValues dv = FieldCache.DEFAULT.GetDocTermOrds(ar, "field"); Assert.AreEqual(3, dv.ValueCount); TermsEnum termsEnum = dv.TermsEnum(); // next() Assert.AreEqual("beer", termsEnum.Next().Utf8ToString()); Assert.AreEqual(0, termsEnum.Ord()); Assert.AreEqual("hello", termsEnum.Next().Utf8ToString()); Assert.AreEqual(1, termsEnum.Ord()); Assert.AreEqual("world", termsEnum.Next().Utf8ToString()); Assert.AreEqual(2, termsEnum.Ord()); // seekCeil() Assert.AreEqual(SeekStatus.NOT_FOUND, termsEnum.SeekCeil(new BytesRef("ha!"))); Assert.AreEqual("hello", termsEnum.Term().Utf8ToString()); Assert.AreEqual(1, termsEnum.Ord()); Assert.AreEqual(SeekStatus.FOUND, termsEnum.SeekCeil(new BytesRef("beer"))); Assert.AreEqual("beer", termsEnum.Term().Utf8ToString()); Assert.AreEqual(0, termsEnum.Ord()); Assert.AreEqual(SeekStatus.END, termsEnum.SeekCeil(new BytesRef("zzz"))); // seekExact() Assert.IsTrue(termsEnum.SeekExact(new BytesRef("beer"))); Assert.AreEqual("beer", termsEnum.Term().Utf8ToString()); Assert.AreEqual(0, termsEnum.Ord()); Assert.IsTrue(termsEnum.SeekExact(new BytesRef("hello"))); Assert.AreEqual("hello", termsEnum.Term().Utf8ToString()); Assert.AreEqual(1, termsEnum.Ord()); Assert.IsTrue(termsEnum.SeekExact(new BytesRef("world"))); Assert.AreEqual("world", termsEnum.Term().Utf8ToString()); Assert.AreEqual(2, termsEnum.Ord()); Assert.IsFalse(termsEnum.SeekExact(new BytesRef("bogus"))); // seek(ord) termsEnum.SeekExact(0); Assert.AreEqual("beer", termsEnum.Term().Utf8ToString()); Assert.AreEqual(0, termsEnum.Ord()); termsEnum.SeekExact(1); Assert.AreEqual("hello", termsEnum.Term().Utf8ToString()); Assert.AreEqual(1, termsEnum.Ord()); termsEnum.SeekExact(2); Assert.AreEqual("world", termsEnum.Term().Utf8ToString()); Assert.AreEqual(2, termsEnum.Ord()); ireader.Dispose(); directory.Dispose(); }
private void Verify(AtomicReader r, int[][] idToOrds, BytesRef[] termsArray, BytesRef prefixRef) { DocTermOrds dto = new DocTermOrds(r, r.LiveDocs, "field", prefixRef, int.MaxValue, TestUtil.NextInt(Random(), 2, 10)); FieldCache.Ints docIDToID = FieldCache.DEFAULT.GetInts(r, "id", false); /* * for(int docID=0;docID<subR.MaxDoc;docID++) { * System.out.println(" docID=" + docID + " id=" + docIDToID[docID]); * } */ if (VERBOSE) { Console.WriteLine("TEST: verify prefix=" + (prefixRef == null ? "null" : prefixRef.Utf8ToString())); Console.WriteLine("TEST: all TERMS:"); TermsEnum allTE = MultiFields.GetTerms(r, "field").Iterator(null); int ord = 0; while (allTE.Next() != null) { Console.WriteLine(" ord=" + (ord++) + " term=" + allTE.Term().Utf8ToString()); } } //final TermsEnum te = subR.Fields.Terms("field").iterator(); TermsEnum te = dto.GetOrdTermsEnum(r); if (dto.NumTerms() == 0) { if (prefixRef == null) { Assert.IsNull(MultiFields.GetTerms(r, "field")); } else { Terms terms = MultiFields.GetTerms(r, "field"); if (terms != null) { TermsEnum termsEnum = terms.Iterator(null); TermsEnum.SeekStatus result = termsEnum.SeekCeil(prefixRef); if (result != TermsEnum.SeekStatus.END) { Assert.IsFalse(StringHelper.StartsWith(termsEnum.Term(), prefixRef), "term=" + termsEnum.Term().Utf8ToString() + " matches prefix=" + prefixRef.Utf8ToString()); } else { // ok } } else { // ok } } return; } if (VERBOSE) { Console.WriteLine("TEST: TERMS:"); te.SeekExact(0); while (true) { Console.WriteLine(" ord=" + te.Ord() + " term=" + te.Term().Utf8ToString()); if (te.Next() == null) { break; } } } SortedSetDocValues iter = dto.GetIterator(r); for (int docID = 0; docID < r.MaxDoc; docID++) { if (VERBOSE) { Console.WriteLine("TEST: docID=" + docID + " of " + r.MaxDoc + " (id=" + docIDToID.Get(docID) + ")"); } iter.Document = docID; int[] answers = idToOrds[docIDToID.Get(docID)]; int upto = 0; long ord; while ((ord = iter.NextOrd()) != SortedSetDocValues.NO_MORE_ORDS) { te.SeekExact(ord); BytesRef expected = termsArray[answers[upto++]]; if (VERBOSE) { Console.WriteLine(" exp=" + expected.Utf8ToString() + " actual=" + te.Term().Utf8ToString()); } Assert.AreEqual(expected, te.Term(), "expected=" + expected.Utf8ToString() + " actual=" + te.Term().Utf8ToString() + " ord=" + ord); } Assert.AreEqual(answers.Length, upto); } }
public override long Ord() { return(@in.Ord()); }
private void AssertEquals(long numOrds, TermsEnum expected, TermsEnum actual) { BytesRef @ref; // sequential next() through all terms while ((@ref = expected.Next()) != null) { Assert.AreEqual(@ref, actual.Next()); Assert.AreEqual(expected.Ord(), actual.Ord()); Assert.AreEqual(expected.Term(), actual.Term()); } Assert.IsNull(actual.Next()); // sequential seekExact(ord) through all terms for (long i = 0; i < numOrds; i++) { expected.SeekExact(i); actual.SeekExact(i); Assert.AreEqual(expected.Ord(), actual.Ord()); Assert.AreEqual(expected.Term(), actual.Term()); } // sequential seekExact(BytesRef) through all terms for (long i = 0; i < numOrds; i++) { expected.SeekExact(i); Assert.IsTrue(actual.SeekExact(expected.Term())); Assert.AreEqual(expected.Ord(), actual.Ord()); Assert.AreEqual(expected.Term(), actual.Term()); } // sequential seekCeil(BytesRef) through all terms for (long i = 0; i < numOrds; i++) { expected.SeekExact(i); Assert.AreEqual(SeekStatus.FOUND, actual.SeekCeil(expected.Term())); Assert.AreEqual(expected.Ord(), actual.Ord()); Assert.AreEqual(expected.Term(), actual.Term()); } // random seekExact(ord) for (long i = 0; i < numOrds; i++) { long randomOrd = TestUtil.NextLong(Random(), 0, numOrds - 1); expected.SeekExact(randomOrd); actual.SeekExact(randomOrd); Assert.AreEqual(expected.Ord(), actual.Ord()); Assert.AreEqual(expected.Term(), actual.Term()); } // random seekExact(BytesRef) for (long i = 0; i < numOrds; i++) { long randomOrd = TestUtil.NextLong(Random(), 0, numOrds - 1); expected.SeekExact(randomOrd); actual.SeekExact(expected.Term()); Assert.AreEqual(expected.Ord(), actual.Ord()); Assert.AreEqual(expected.Term(), actual.Term()); } // random seekCeil(BytesRef) for (long i = 0; i < numOrds; i++) { BytesRef target = new BytesRef(TestUtil.RandomUnicodeString(Random())); SeekStatus expectedStatus = expected.SeekCeil(target); Assert.AreEqual(expectedStatus, actual.SeekCeil(target)); if (expectedStatus != SeekStatus.END) { Assert.AreEqual(expected.Ord(), actual.Ord()); Assert.AreEqual(expected.Term(), actual.Term()); } } }