internal static DocMap Build(int maxDoc, IBits liveDocs) { if (Debugging.AssertsEnabled) { Debugging.Assert(liveDocs != null); } MonotonicAppendingInt64Buffer docMap = new MonotonicAppendingInt64Buffer(); int del = 0; for (int i = 0; i < maxDoc; ++i) { docMap.Add(i - del); if (!liveDocs.Get(i)) { ++del; } } docMap.Freeze(); int numDeletedDocs = del; if (Debugging.AssertsEnabled) { Debugging.Assert(docMap.Count == maxDoc); } return(new DocMapAnonymousInnerClassHelper(maxDoc, liveDocs, docMap, numDeletedDocs)); }
private static MonotonicAppendingInt64Buffer LoadSingleZeroBuffer() // LUCENENET: Avoid static constructors (see https://github.com/apache/lucenenet/pull/224#issuecomment-469284006) { var buffer = new MonotonicAppendingInt64Buffer(1, 64, PackedInt32s.COMPACT); buffer.Add(0L); buffer.Freeze(); return(buffer); }
private static MonotonicAppendingInt64Buffer LoadSingleZeroBuffer() { var buffer = new MonotonicAppendingInt64Buffer(0, 64, PackedInt32s.COMPACT); buffer.Add(0); buffer.Freeze(); return(buffer); }
/// <summary> /// Computes the old-to-new permutation over the given comparer. /// </summary> private static Sorter.DocMap Sort(int maxDoc, DocComparer comparer) { // check if the index is sorted bool sorted = true; for (int i = 1; i < maxDoc; ++i) { if (comparer.Compare(i - 1, i) > 0) { sorted = false; break; } } if (sorted) { return(null); } // sort doc IDs int[] docs = new int[maxDoc]; for (int i = 0; i < maxDoc; i++) { docs[i] = i; } DocValueSorter sorter = new DocValueSorter(docs, comparer); // It can be common to sort a reader, add docs, sort it again, ... and in // that case timSort can save a lot of time sorter.Sort(0, docs.Length); // docs is now the newToOld mapping // The reason why we use MonotonicAppendingLongBuffer here is that it // wastes very little memory if the index is in random order but can save // a lot of memory if the index is already "almost" sorted MonotonicAppendingInt64Buffer newToOld = new MonotonicAppendingInt64Buffer(); for (int i = 0; i < maxDoc; ++i) { newToOld.Add(docs[i]); } newToOld.Freeze(); for (int i = 0; i < maxDoc; ++i) { docs[(int)newToOld.Get(i)] = i; } // docs is now the oldToNew mapping MonotonicAppendingInt64Buffer oldToNew = new MonotonicAppendingInt64Buffer(); for (int i = 0; i < maxDoc; ++i) { oldToNew.Add(docs[i]); } oldToNew.Freeze(); return(new DocMapAnonymousClass(maxDoc, newToOld, oldToNew)); }
/// <summary> /// Creates an ordinal map that allows mapping ords to/from a merged /// space from <c>subs</c>. </summary> /// <param name="owner"> a cache key </param> /// <param name="subs"> <see cref="TermsEnum"/>s that support <see cref="TermsEnum.Ord"/>. They need /// not be dense (e.g. can be FilteredTermsEnums). </param> /// <exception cref="System.IO.IOException"> if an I/O error occurred. </exception> public OrdinalMap(object owner, TermsEnum[] subs) { // create the ordinal mappings by pulling a termsenum over each sub's // unique terms, and walking a multitermsenum over those this.owner = owner; globalOrdDeltas = new MonotonicAppendingInt64Buffer(PackedInt32s.COMPACT); firstSegments = new AppendingPackedInt64Buffer(PackedInt32s.COMPACT); ordDeltas = new MonotonicAppendingInt64Buffer[subs.Length]; for (int i = 0; i < ordDeltas.Length; i++) { ordDeltas[i] = new MonotonicAppendingInt64Buffer(); } long[] segmentOrds = new long[subs.Length]; ReaderSlice[] slices = new ReaderSlice[subs.Length]; TermsEnumIndex[] indexes = new TermsEnumIndex[slices.Length]; for (int i = 0; i < slices.Length; i++) { slices[i] = new ReaderSlice(0, 0, i); indexes[i] = new TermsEnumIndex(subs[i], i); } MultiTermsEnum mte = new MultiTermsEnum(slices); mte.Reset(indexes); long globalOrd = 0; while (mte.Next() != null) { TermsEnumWithSlice[] matches = mte.MatchArray; for (int i = 0; i < mte.MatchCount; i++) { int segmentIndex = matches[i].Index; long segmentOrd = matches[i].Terms.Ord; long delta = globalOrd - segmentOrd; // for each unique term, just mark the first segment index/delta where it occurs if (i == 0) { firstSegments.Add(segmentIndex); globalOrdDeltas.Add(delta); } // for each per-segment ord, map it back to the global term. while (segmentOrds[segmentIndex] <= segmentOrd) { ordDeltas[segmentIndex].Add(delta); segmentOrds[segmentIndex]++; } } globalOrd++; } firstSegments.Freeze(); globalOrdDeltas.Freeze(); for (int i = 0; i < ordDeltas.Length; ++i) { ordDeltas[i].Freeze(); } }
/// <summary> /// Build the <see cref="PForDeltaDocIdSet"/> instance. </summary> public virtual PForDeltaDocIdSet Build() { if (Debugging.AssertsEnabled) { Debugging.Assert(bufferSize < BLOCK_SIZE); } if (cardinality == 0) { if (Debugging.AssertsEnabled) { Debugging.Assert(previousDoc == -1); } return(EMPTY); } EncodeBlock(); var dataArr = Arrays.CopyOf(data.Bytes, data.Length + MAX_BYTE_BLOCK_COUNT); int indexSize = (numBlocks - 1) / indexInterval + 1; MonotonicAppendingInt64Buffer docIDs, offsets; if (indexSize <= 1) { docIDs = offsets = SINGLE_ZERO_BUFFER; } else { const int pageSize = 128; int initialPageCount = (indexSize + pageSize - 1) / pageSize; docIDs = new MonotonicAppendingInt64Buffer(initialPageCount, pageSize, PackedInt32s.COMPACT); offsets = new MonotonicAppendingInt64Buffer(initialPageCount, pageSize, PackedInt32s.COMPACT); // Now build the index Iterator it = new Iterator(dataArr, cardinality, int.MaxValue, SINGLE_ZERO_BUFFER, SINGLE_ZERO_BUFFER); for (int k = 0; k < indexSize; ++k) { docIDs.Add(it.DocID + 1); offsets.Add(it.offset); for (int i = 0; i < indexInterval; ++i) { it.SkipBlock(); if (it.DocID == DocIdSetIterator.NO_MORE_DOCS) { goto indexBreak; } } //indexContinue: ; } indexBreak: docIDs.Freeze(); offsets.Freeze(); } return(new PForDeltaDocIdSet(dataArr, cardinality, indexInterval, docIDs, offsets)); }
/// <summary> /// Build a new <see cref="WAH8DocIdSet"/>. </summary> public virtual WAH8DocIdSet Build() { if (cardinality == 0) { Debug.Assert(lastWordNum == -1); return(EMPTY); } WriteSequence(); byte[] data = Arrays.CopyOf(@out.Bytes, @out.Length); // Now build the index int valueCount = (numSequences - 1) / indexInterval + 1; MonotonicAppendingInt64Buffer indexPositions, indexWordNums; if (valueCount <= 1) { indexPositions = indexWordNums = SINGLE_ZERO_BUFFER; } else { const int pageSize = 128; int initialPageCount = (valueCount + pageSize - 1) / pageSize; MonotonicAppendingInt64Buffer positions = new MonotonicAppendingInt64Buffer(initialPageCount, pageSize, PackedInt32s.COMPACT); MonotonicAppendingInt64Buffer wordNums = new MonotonicAppendingInt64Buffer(initialPageCount, pageSize, PackedInt32s.COMPACT); positions.Add(0L); wordNums.Add(0L); Iterator it = new Iterator(data, cardinality, int.MaxValue, SINGLE_ZERO_BUFFER, SINGLE_ZERO_BUFFER); Debug.Assert([email protected] == 0); Debug.Assert(it.wordNum == -1); for (int i = 1; i < valueCount; ++i) { // skip indexInterval sequences for (int j = 0; j < indexInterval; ++j) { bool readSequence = it.ReadSequence(); Debug.Assert(readSequence); it.SkipDirtyBytes(); } int position = [email protected]; int wordNum = it.wordNum; positions.Add(position); wordNums.Add(wordNum + 1); } positions.Freeze(); wordNums.Freeze(); indexPositions = positions; indexWordNums = wordNums; } return(new WAH8DocIdSet(data, cardinality, indexInterval, indexPositions, indexWordNums)); }
static PForDeltaDocIdSet() { SINGLE_ZERO_BUFFER.Add(0); SINGLE_ZERO_BUFFER.Freeze(); int maxByteBLockCount = 0; for (int i = 1; i < ITERATIONS.Length; ++i) { DECODERS[i] = PackedInt32s.GetDecoder(PackedInt32s.Format.PACKED, PackedInt32s.VERSION_CURRENT, i); Debug.Assert(BLOCK_SIZE % DECODERS[i].ByteValueCount == 0); ITERATIONS[i] = BLOCK_SIZE / DECODERS[i].ByteValueCount; BYTE_BLOCK_COUNTS[i] = ITERATIONS[i] * DECODERS[i].ByteBlockCount; maxByteBLockCount = Math.Max(maxByteBLockCount, DECODERS[i].ByteBlockCount); } MAX_BYTE_BLOCK_COUNT = maxByteBLockCount; }
internal virtual MonotonicAppendingInt64Buffer GetDeletes(IList <AtomicReader> readers) { MonotonicAppendingInt64Buffer deletes = new MonotonicAppendingInt64Buffer(); int deleteCount = 0; foreach (AtomicReader reader in readers) { int maxDoc = reader.MaxDoc; IBits liveDocs = reader.LiveDocs; for (int i = 0; i < maxDoc; ++i) { if (liveDocs != null && !liveDocs.Get(i)) { ++deleteCount; } else { deletes.Add(deleteCount); } } } deletes.Freeze(); return(deletes); }
static WAH8DocIdSet() { SINGLE_ZERO_BUFFER.Add(0L); SINGLE_ZERO_BUFFER.Freeze(); }