public override MergeSpecification FindForcedDeletesMerges(SegmentInfos infos) { if (Verbose()) { Message("findForcedDeletesMerges infos=" + Writer.Get().SegString(infos.Segments) + " forceMergeDeletesPctAllowed=" + ForceMergeDeletesPctAllowed_Renamed); } List <SegmentCommitInfo> eligible = new List <SegmentCommitInfo>(); ICollection <SegmentCommitInfo> merging = Writer.Get().MergingSegments; foreach (SegmentCommitInfo info in infos.Segments) { double pctDeletes = 100.0 * ((double)Writer.Get().NumDeletedDocs(info)) / info.Info.DocCount; if (pctDeletes > ForceMergeDeletesPctAllowed_Renamed && !merging.Contains(info)) { eligible.Add(info); } } if (eligible.Count == 0) { return(null); } eligible.Sort(new SegmentByteSizeDescending(this)); if (Verbose()) { Message("eligible=" + eligible); } int start = 0; MergeSpecification spec = null; while (start < eligible.Count) { // Don't enforce max merged size here: app is explicitly // calling forceMergeDeletes, and knows this may take a // long time / produce big segments (like forceMerge): int end = Math.Min(start + MaxMergeAtOnceExplicit_Renamed, eligible.Count); if (spec == null) { spec = new MergeSpecification(); } OneMerge merge = new OneMerge(eligible.SubList(start, end)); if (Verbose()) { Message("add merge=" + Writer.Get().SegString(merge.Segments)); } spec.Add(merge); start = end; } return(spec); }
public override MergeSpecification FindMerges(MergeTrigger mergeTrigger, SegmentInfos segmentInfos) { MergeSpecification ms = new MergeSpecification(); if (doMerge) { OneMerge om = new OneMerge(segmentInfos.AsList().GetView(start, length)); // LUCENENET: Converted end index to length ms.Add(om); doMerge = false; return(ms); } return(null); }
public override MergeSpecification FindMerges(MergeTrigger mergeTrigger, SegmentInfos segmentInfos) { MergeSpecification ms = new MergeSpecification(); if (DoMerge) { OneMerge om = new OneMerge(segmentInfos.AsList().SubList(Start, Start + Length)); ms.Add(om); DoMerge = false; return(ms); } return(null); }
public virtual void Add(OneMerge merge) { merges.Add(merge); }
public virtual void Add(OneMerge merge) { merges.Add(merge); }
public override MergeSpecification FindMerges(MergeTrigger?mergeTrigger, SegmentInfos infos) { if (Verbose()) { Message("findMerges: " + infos.Size() + " segments"); } if (infos.Size() == 0) { return(null); } ICollection <SegmentCommitInfo> merging = Writer.Get().MergingSegments; ICollection <SegmentCommitInfo> toBeMerged = new HashSet <SegmentCommitInfo>(); List <SegmentCommitInfo> infosSorted = new List <SegmentCommitInfo>(infos.AsList()); infosSorted.Sort(new SegmentByteSizeDescending(this)); // Compute total index bytes & print details about the index long totIndexBytes = 0; long minSegmentBytes = long.MaxValue; foreach (SegmentCommitInfo info in infosSorted) { long segBytes = Size(info); if (Verbose()) { string extra = merging.Contains(info) ? " [merging]" : ""; if (segBytes >= MaxMergedSegmentBytes / 2.0) { extra += " [skip: too large]"; } else if (segBytes < FloorSegmentBytes) { extra += " [floored]"; } Message(" seg=" + Writer.Get().SegString(info) + " size=" + String.Format(CultureInfo.InvariantCulture, "{0:0.00}", segBytes / 1024 / 1024.0) + " MB" + extra); } minSegmentBytes = Math.Min(segBytes, minSegmentBytes); // Accum total byte size totIndexBytes += segBytes; } // If we have too-large segments, grace them out // of the maxSegmentCount: int tooBigCount = 0; while (tooBigCount < infosSorted.Count && Size(infosSorted[tooBigCount]) >= MaxMergedSegmentBytes / 2.0) { totIndexBytes -= Size(infosSorted[tooBigCount]); tooBigCount++; } minSegmentBytes = FloorSize(minSegmentBytes); // Compute max allowed segs in the index long levelSize = minSegmentBytes; long bytesLeft = totIndexBytes; double allowedSegCount = 0; while (true) { double segCountLevel = bytesLeft / (double)levelSize; if (segCountLevel < SegsPerTier) { allowedSegCount += Math.Ceiling(segCountLevel); break; } allowedSegCount += SegsPerTier; bytesLeft -= (long)(SegsPerTier * levelSize); levelSize *= MaxMergeAtOnce_Renamed; } int allowedSegCountInt = (int)allowedSegCount; MergeSpecification spec = null; // Cycle to possibly select more than one merge: while (true) { long mergingBytes = 0; // Gather eligible segments for merging, ie segments // not already being merged and not already picked (by // prior iteration of this loop) for merging: IList <SegmentCommitInfo> eligible = new List <SegmentCommitInfo>(); for (int idx = tooBigCount; idx < infosSorted.Count; idx++) { SegmentCommitInfo info = infosSorted[idx]; if (merging.Contains(info)) { mergingBytes += info.SizeInBytes(); } else if (!toBeMerged.Contains(info)) { eligible.Add(info); } } bool maxMergeIsRunning = mergingBytes >= MaxMergedSegmentBytes; if (Verbose()) { Message(" allowedSegmentCount=" + allowedSegCountInt + " vs count=" + infosSorted.Count + " (eligible count=" + eligible.Count + ") tooBigCount=" + tooBigCount); } if (eligible.Count == 0) { return(spec); } if (eligible.Count >= allowedSegCountInt) { // OK we are over budget -- find best merge! MergeScore bestScore = null; IList <SegmentCommitInfo> best = null; bool bestTooLarge = false; long bestMergeBytes = 0; // Consider all merge starts: for (int startIdx = 0; startIdx <= eligible.Count - MaxMergeAtOnce_Renamed; startIdx++) { long totAfterMergeBytes = 0; IList <SegmentCommitInfo> candidate = new List <SegmentCommitInfo>(); bool hitTooLarge = false; for (int idx = startIdx; idx < eligible.Count && candidate.Count < MaxMergeAtOnce_Renamed; idx++) { SegmentCommitInfo info = eligible[idx]; long segBytes = Size(info); if (totAfterMergeBytes + segBytes > MaxMergedSegmentBytes) { hitTooLarge = true; // NOTE: we continue, so that we can try // "packing" smaller segments into this merge // to see if we can get closer to the max // size; this in general is not perfect since // this is really "bin packing" and we'd have // to try different permutations. continue; } candidate.Add(info); totAfterMergeBytes += segBytes; } MergeScore score = Score(candidate, hitTooLarge, mergingBytes); if (Verbose()) { Message(" maybe=" + Writer.Get().SegString(candidate) + " score=" + score.Score + " " + score.Explanation + " tooLarge=" + hitTooLarge + " size=" + string.Format(CultureInfo.InvariantCulture, "%.3f MB", totAfterMergeBytes / 1024.0 / 1024.0)); } // If we are already running a max sized merge // (maxMergeIsRunning), don't allow another max // sized merge to kick off: if ((bestScore == null || score.Score < bestScore.Score) && (!hitTooLarge || !maxMergeIsRunning)) { best = candidate; bestScore = score; bestTooLarge = hitTooLarge; bestMergeBytes = totAfterMergeBytes; } } if (best != null) { if (spec == null) { spec = new MergeSpecification(); } OneMerge merge = new OneMerge(best); spec.Add(merge); foreach (SegmentCommitInfo info in merge.Segments) { toBeMerged.Add(info); } if (Verbose()) { Message(" add merge=" + Writer.Get().SegString(merge.Segments) + " size=" + string.Format(CultureInfo.InvariantCulture, "%.3f MB", bestMergeBytes / 1024.0 / 1024.0) + " score=" + string.Format(CultureInfo.InvariantCulture, "%.3f", bestScore.Score) + " " + bestScore.Explanation + (bestTooLarge ? " [max merge]" : "")); } } else { return(spec); } } else { return(spec); } } }
public override MergeSpecification FindForcedMerges(SegmentInfos infos, int maxSegmentCount, IDictionary <SegmentCommitInfo, bool?> segmentsToMerge) { if (Verbose()) { Message("findForcedMerges maxSegmentCount=" + maxSegmentCount + " infos=" + Writer.Get().SegString(infos.Segments) + " segmentsToMerge=" + segmentsToMerge); } List <SegmentCommitInfo> eligible = new List <SegmentCommitInfo>(); bool forceMergeRunning = false; ICollection <SegmentCommitInfo> merging = Writer.Get().MergingSegments; bool?segmentIsOriginal = false; foreach (SegmentCommitInfo info in infos.Segments) { bool?isOriginal = segmentsToMerge[info]; if (isOriginal != null) { segmentIsOriginal = isOriginal; if (!merging.Contains(info)) { eligible.Add(info); } else { forceMergeRunning = true; } } } if (eligible.Count == 0) { return(null); } if ((maxSegmentCount > 1 && eligible.Count <= maxSegmentCount) || (maxSegmentCount == 1 && eligible.Count == 1 && (segmentIsOriginal == false || IsMerged(infos, eligible[0])))) { if (Verbose()) { Message("already merged"); } return(null); } eligible.Sort(new SegmentByteSizeDescending(this)); if (Verbose()) { Message("eligible=" + eligible); Message("forceMergeRunning=" + forceMergeRunning); } int end = eligible.Count; MergeSpecification spec = null; // Do full merges, first, backwards: while (end >= MaxMergeAtOnceExplicit_Renamed + maxSegmentCount - 1) { if (spec == null) { spec = new MergeSpecification(); } OneMerge merge = new OneMerge(eligible.SubList(end - MaxMergeAtOnceExplicit_Renamed, end)); if (Verbose()) { Message("add merge=" + Writer.Get().SegString(merge.Segments)); } spec.Add(merge); end -= MaxMergeAtOnceExplicit_Renamed; } if (spec == null && !forceMergeRunning) { // Do final merge int numToMerge = end - maxSegmentCount + 1; OneMerge merge = new OneMerge(eligible.SubList(end - numToMerge, end)); if (Verbose()) { Message("add final merge=" + merge.SegString(Writer.Get().Directory)); } spec = new MergeSpecification(); spec.Add(merge); } return(spec); }
public DocMapAnonymousInnerClassHelper(OneMerge outerInstance) { this.OuterInstance = outerInstance; }
public override void Add(OneMerge merge) { base.Add(new SortingOneMerge(outerInstance, merge.Segments)); }
public DocMapAnonymousInnerClassHelper(OneMerge outerInstance) { this.OuterInstance = outerInstance; }
public override MergeSpecification FindMerges(MergeTrigger? mergeTrigger, SegmentInfos segmentInfos) { MergeSpecification ms = new MergeSpecification(); if (DoMerge) { OneMerge om = new OneMerge(segmentInfos.AsList().SubList(Start, Start + Length)); ms.Add(om); DoMerge = false; return ms; } return null; }
public override void add(OneMerge merge) { base.add(new SortingOneMerge(outerInstance, merge.segments)); }