public override MergeSpecification FindMerges(MergeTrigger mergeTrigger, SegmentInfos infos)
{
if (Verbose())
{
Message("findMerges: " + infos.Count + " segments");
}
if (infos.Count == 0)
{
return(null);
}
ICollection <SegmentCommitInfo> merging = m_writer.Get().MergingSegments;
ICollection <SegmentCommitInfo> toBeMerged = new HashSet <SegmentCommitInfo>();
List <SegmentCommitInfo> infosSorted = new List <SegmentCommitInfo>(infos.AsList());
infosSorted.Sort(new SegmentByteSizeDescending(this));
// Compute total index bytes & print details about the index
long totIndexBytes = 0;
long minSegmentBytes = long.MaxValue;
foreach (SegmentCommitInfo info in infosSorted)
{
long segBytes = Size(info);
if (Verbose())
{
string extra = merging.Contains(info) ? " [merging]" : "";
if (segBytes >= maxMergedSegmentBytes / 2.0)
{
extra += " [skip: too large]";
}
else if (segBytes < floorSegmentBytes)
{
extra += " [floored]";
}
Message(" seg=" + m_writer.Get().SegString(info) + " size=" + string.Format("{0:0.000}", segBytes / 1024 / 1024.0) + " MB" + extra);
}
minSegmentBytes = Math.Min(segBytes, minSegmentBytes);
// Accum total byte size
totIndexBytes += segBytes;
}
// If we have too-large segments, grace them out
// of the maxSegmentCount:
int tooBigCount = 0;
while (tooBigCount < infosSorted.Count && Size(infosSorted[tooBigCount]) >= maxMergedSegmentBytes / 2.0)
{
totIndexBytes -= Size(infosSorted[tooBigCount]);
tooBigCount++;
}
minSegmentBytes = FloorSize(minSegmentBytes);
// Compute max allowed segs in the index
long levelSize = minSegmentBytes;
long bytesLeft = totIndexBytes;
double allowedSegCount = 0;
while (true)
{
double segCountLevel = bytesLeft / (double)levelSize;
if (segCountLevel < segsPerTier)
{
allowedSegCount += Math.Ceiling(segCountLevel);
break;
}
allowedSegCount += segsPerTier;
bytesLeft -= (long)(segsPerTier * levelSize);
levelSize *= maxMergeAtOnce;
}
int allowedSegCountInt = (int)allowedSegCount;
MergeSpecification spec = null;
// Cycle to possibly select more than one merge:
while (true)
{
long mergingBytes = 0;
// Gather eligible segments for merging, ie segments
// not already being merged and not already picked (by
// prior iteration of this loop) for merging:
IList <SegmentCommitInfo> eligible = new List <SegmentCommitInfo>();
for (int idx = tooBigCount; idx < infosSorted.Count; idx++)
{
SegmentCommitInfo info = infosSorted[idx];
if (merging.Contains(info))
{
mergingBytes += info.GetSizeInBytes();
}
else if (!toBeMerged.Contains(info))
{
eligible.Add(info);
}
}
bool maxMergeIsRunning = mergingBytes >= maxMergedSegmentBytes;
if (Verbose())
{
Message(" allowedSegmentCount=" + allowedSegCountInt + " vs count=" + infosSorted.Count + " (eligible count=" + eligible.Count + ") tooBigCount=" + tooBigCount);
}
if (eligible.Count == 0)
{
return(spec);
}
if (eligible.Count >= allowedSegCountInt)
{
// OK we are over budget -- find best merge!
MergeScore bestScore = null;
IList <SegmentCommitInfo> best = null;
bool bestTooLarge = false;
long bestMergeBytes = 0;
// Consider all merge starts:
for (int startIdx = 0; startIdx <= eligible.Count - maxMergeAtOnce; startIdx++)
{
long totAfterMergeBytes = 0;
IList <SegmentCommitInfo> candidate = new List <SegmentCommitInfo>();
bool hitTooLarge = false;
for (int idx = startIdx; idx < eligible.Count && candidate.Count < maxMergeAtOnce; idx++)
{
SegmentCommitInfo info = eligible[idx];
long segBytes = Size(info);
if (totAfterMergeBytes + segBytes > maxMergedSegmentBytes)
{
hitTooLarge = true;
// NOTE: we continue, so that we can try
// "packing" smaller segments into this merge
// to see if we can get closer to the max
// size; this in general is not perfect since
// this is really "bin packing" and we'd have
// to try different permutations.
continue;
}
candidate.Add(info);
totAfterMergeBytes += segBytes;
}
MergeScore score = Score(candidate, hitTooLarge, mergingBytes);
if (Verbose())
{
Message(" maybe=" + m_writer.Get().SegString(candidate) + " score=" + score.Score + " " + score.Explanation + " tooLarge=" + hitTooLarge + " size=" + string.Format("{0:0.000} MB", totAfterMergeBytes / 1024.0 / 1024.0));
}
// If we are already running a max sized merge
// (maxMergeIsRunning), don't allow another max
// sized merge to kick off:
if ((bestScore == null || score.Score < bestScore.Score) && (!hitTooLarge || !maxMergeIsRunning))
{
best = candidate;
bestScore = score;
bestTooLarge = hitTooLarge;
bestMergeBytes = totAfterMergeBytes;
}
}
if (best != null)
{
if (spec == null)
{
spec = new MergeSpecification();
}
OneMerge merge = new OneMerge(best);
spec.Add(merge);
foreach (SegmentCommitInfo info in merge.Segments)
{
toBeMerged.Add(info);
}
if (Verbose())
{
Message(" add merge=" + m_writer.Get().SegString(merge.Segments) + " size=" + string.Format("{0:0.000} MB", bestMergeBytes / 1024.0 / 1024.0) + " score=" + string.Format("{0:0.000}", bestScore.Score) + " " + bestScore.Explanation + (bestTooLarge ? " [max merge]" : ""));
}
}
else
{
return(spec);
}
}
else
{
return(spec);
}
}
}
public virtual ApplyDeletesResult ApplyDeletesAndUpdates(IndexWriter.ReaderPool readerPool, IList <SegmentCommitInfo> infos)
{
lock (this)
{
long t0 = Environment.TickCount;
if (infos.Count == 0)
{
return(new ApplyDeletesResult(false, nextGen++, null));
}
if (Debugging.AssertsEnabled)
{
Debugging.Assert(CheckDeleteStats());
}
if (!Any())
{
if (infoStream.IsEnabled("BD"))
{
infoStream.Message("BD", "applyDeletes: no deletes; skipping");
}
return(new ApplyDeletesResult(false, nextGen++, null));
}
if (infoStream.IsEnabled("BD"))
{
infoStream.Message("BD", "applyDeletes: infos=" + string.Format(J2N.Text.StringFormatter.InvariantCulture, "{0}", infos) + " packetCount=" + updates.Count);
}
long gen = nextGen++;
List <SegmentCommitInfo> infos2 = new List <SegmentCommitInfo>();
infos2.AddRange(infos);
infos2.Sort(sortSegInfoByDelGen);
CoalescedUpdates coalescedUpdates = null;
bool anyNewDeletes = false;
int infosIDX = infos2.Count - 1;
int delIDX = updates.Count - 1;
IList <SegmentCommitInfo> allDeleted = null;
while (infosIDX >= 0)
{
//System.out.println("BD: cycle delIDX=" + delIDX + " infoIDX=" + infosIDX);
FrozenBufferedUpdates packet = delIDX >= 0 ? updates[delIDX] : null;
SegmentCommitInfo info = infos2[infosIDX];
long segGen = info.BufferedDeletesGen;
if (packet != null && segGen < packet.DelGen)
{
// System.out.println(" coalesce");
if (coalescedUpdates == null)
{
coalescedUpdates = new CoalescedUpdates();
}
if (!packet.isSegmentPrivate)
{
/*
* Only coalesce if we are NOT on a segment private del packet: the segment private del packet
* must only applied to segments with the same delGen. Yet, if a segment is already deleted
* from the SI since it had no more documents remaining after some del packets younger than
* its segPrivate packet (higher delGen) have been applied, the segPrivate packet has not been
* removed.
*/
coalescedUpdates.Update(packet);
}
delIDX--;
}
else if (packet != null && segGen == packet.DelGen)
{
if (Debugging.AssertsEnabled)
{
Debugging.Assert(packet.isSegmentPrivate, "Packet and Segments deletegen can only match on a segment private del packet gen={0}", segGen);
}
//System.out.println(" eq");
// Lock order: IW -> BD -> RP
if (Debugging.AssertsEnabled)
{
Debugging.Assert(readerPool.InfoIsLive(info));
}
ReadersAndUpdates rld = readerPool.Get(info, true);
SegmentReader reader = rld.GetReader(IOContext.READ);
int delCount = 0;
bool segAllDeletes;
try
{
DocValuesFieldUpdates.Container dvUpdates = new DocValuesFieldUpdates.Container();
if (coalescedUpdates != null)
{
//System.out.println(" del coalesced");
delCount += (int)ApplyTermDeletes(coalescedUpdates.TermsIterable(), rld, reader);
delCount += (int)ApplyQueryDeletes(coalescedUpdates.QueriesIterable(), rld, reader);
ApplyDocValuesUpdates(coalescedUpdates.numericDVUpdates, rld, reader, dvUpdates);
ApplyDocValuesUpdates(coalescedUpdates.binaryDVUpdates, rld, reader, dvUpdates);
}
//System.out.println(" del exact");
// Don't delete by Term here; DocumentsWriterPerThread
// already did that on flush:
delCount += (int)ApplyQueryDeletes(packet.GetQueriesEnumerable(), rld, reader);
ApplyDocValuesUpdates(packet.numericDVUpdates, rld, reader, dvUpdates);
ApplyDocValuesUpdates(packet.binaryDVUpdates, rld, reader, dvUpdates);
if (dvUpdates.Any())
{
rld.WriteFieldUpdates(info.Info.Dir, dvUpdates);
}
int fullDelCount = rld.Info.DelCount + rld.PendingDeleteCount;
if (Debugging.AssertsEnabled)
{
Debugging.Assert(fullDelCount <= rld.Info.Info.DocCount);
}
segAllDeletes = fullDelCount == rld.Info.Info.DocCount;
}
finally
{
rld.Release(reader);
readerPool.Release(rld);
}
anyNewDeletes |= delCount > 0;
if (segAllDeletes)
{
if (allDeleted == null)
{
allDeleted = new List <SegmentCommitInfo>();
}
allDeleted.Add(info);
}
if (infoStream.IsEnabled("BD"))
{
infoStream.Message("BD", "seg=" + info + " segGen=" + segGen + " segDeletes=[" + packet + "]; coalesced deletes=[" + (coalescedUpdates == null ? "null" : coalescedUpdates.ToString()) + "] newDelCount=" + delCount + (segAllDeletes ? " 100% deleted" : ""));
}
if (coalescedUpdates == null)
{
coalescedUpdates = new CoalescedUpdates();
}
/*
* Since we are on a segment private del packet we must not
* update the coalescedDeletes here! We can simply advance to the
* next packet and seginfo.
*/
delIDX--;
infosIDX--;
info.SetBufferedDeletesGen(gen);
}
else
{
//System.out.println(" gt");
if (coalescedUpdates != null)
{
// Lock order: IW -> BD -> RP
if (Debugging.AssertsEnabled)
{
Debugging.Assert(readerPool.InfoIsLive(info));
}
ReadersAndUpdates rld = readerPool.Get(info, true);
SegmentReader reader = rld.GetReader(IOContext.READ);
int delCount = 0;
bool segAllDeletes;
try
{
delCount += (int)ApplyTermDeletes(coalescedUpdates.TermsIterable(), rld, reader);
delCount += (int)ApplyQueryDeletes(coalescedUpdates.QueriesIterable(), rld, reader);
DocValuesFieldUpdates.Container dvUpdates = new DocValuesFieldUpdates.Container();
ApplyDocValuesUpdates(coalescedUpdates.numericDVUpdates, rld, reader, dvUpdates);
ApplyDocValuesUpdates(coalescedUpdates.binaryDVUpdates, rld, reader, dvUpdates);
if (dvUpdates.Any())
{
rld.WriteFieldUpdates(info.Info.Dir, dvUpdates);
}
int fullDelCount = rld.Info.DelCount + rld.PendingDeleteCount;
if (Debugging.AssertsEnabled)
{
Debugging.Assert(fullDelCount <= rld.Info.Info.DocCount);
}
segAllDeletes = fullDelCount == rld.Info.Info.DocCount;
}
finally
{
rld.Release(reader);
readerPool.Release(rld);
}
anyNewDeletes |= delCount > 0;
if (segAllDeletes)
{
if (allDeleted == null)
{
allDeleted = new List <SegmentCommitInfo>();
}
allDeleted.Add(info);
}
if (infoStream.IsEnabled("BD"))
{
infoStream.Message("BD", "seg=" + info + " segGen=" + segGen + " coalesced deletes=[" + coalescedUpdates + "] newDelCount=" + delCount + (segAllDeletes ? " 100% deleted" : ""));
}
}
info.SetBufferedDeletesGen(gen);
infosIDX--;
}
}
if (Debugging.AssertsEnabled)
{
Debugging.Assert(CheckDeleteStats());
}
if (infoStream.IsEnabled("BD"))
{
infoStream.Message("BD", "applyDeletes took " + (Environment.TickCount - t0) + " msec");
}
// assert infos != segmentInfos || !any() : "infos=" + infos + " segmentInfos=" + segmentInfos + " any=" + any;
return(new ApplyDeletesResult(anyNewDeletes, gen, allDeleted));
}
}
public SegmentInfoAndLevel(SegmentCommitInfo info, float level, int index)
{
this.info = info;
this.level = level;
this.index = index;
}
/// <summary>
/// Checks if any merges are now necessary and returns a
/// <see cref="MergePolicy.MergeSpecification"/> if so. A merge
/// is necessary when there are more than
/// <see cref="MergeFactor"/> segments at a given level. When
/// multiple levels have too many segments, this method
/// will return multiple merges, allowing the
/// <see cref="MergeScheduler"/> to use concurrency.
/// </summary>
public override MergeSpecification FindMerges(MergeTrigger mergeTrigger, SegmentInfos infos)
{
int numSegments = infos.Count;
if (IsVerbose)
{
Message("findMerges: " + numSegments + " segments");
}
// Compute levels, which is just log (base mergeFactor)
// of the size of each segment
IList <SegmentInfoAndLevel> levels = new List <SegmentInfoAndLevel>();
var norm = (float)Math.Log(m_mergeFactor);
ICollection <SegmentCommitInfo> mergingSegments = m_writer.Get().MergingSegments;
for (int i = 0; i < numSegments; i++)
{
SegmentCommitInfo info = infos.Info(i);
long size = Size(info);
// Floor tiny segments
if (size < 1)
{
size = 1;
}
SegmentInfoAndLevel infoLevel = new SegmentInfoAndLevel(info, (float)Math.Log(size) / norm, i);
levels.Add(infoLevel);
if (IsVerbose)
{
long segBytes = SizeBytes(info);
string extra = mergingSegments.Contains(info) ? " [merging]" : "";
if (size >= m_maxMergeSize)
{
extra += " [skip: too large]";
}
Message("seg=" + m_writer.Get().SegString(info) + " level=" + infoLevel.level + " size=" + String.Format(CultureInfo.InvariantCulture, "{0:0.00} MB", segBytes / 1024 / 1024.0) + extra);
}
}
float levelFloor;
if (m_minMergeSize <= 0)
{
levelFloor = (float)0.0;
}
else
{
levelFloor = (float)(Math.Log(m_minMergeSize) / norm);
}
// Now, we quantize the log values into levels. The
// first level is any segment whose log size is within
// LEVEL_LOG_SPAN of the max size, or, who has such as
// segment "to the right". Then, we find the max of all
// other segments and use that to define the next level
// segment, etc.
MergeSpecification spec = null;
int numMergeableSegments = levels.Count;
int start = 0;
while (start < numMergeableSegments)
{
// Find max level of all segments not already
// quantized.
float maxLevel = levels[start].level;
for (int i = 1 + start; i < numMergeableSegments; i++)
{
float level = levels[i].level;
if (level > maxLevel)
{
maxLevel = level;
}
}
// Now search backwards for the rightmost segment that
// falls into this level:
float levelBottom;
if (maxLevel <= levelFloor)
{
// All remaining segments fall into the min level
levelBottom = -1.0F;
}
else
{
levelBottom = (float)(maxLevel - LEVEL_LOG_SPAN);
// Force a boundary at the level floor
if (levelBottom < levelFloor && maxLevel >= levelFloor)
{
levelBottom = levelFloor;
}
}
int upto = numMergeableSegments - 1;
while (upto >= start)
{
if (levels[upto].level >= levelBottom)
{
break;
}
upto--;
}
if (IsVerbose)
{
Message(" level " + levelBottom.ToString("0.0") + " to " + maxLevel.ToString("0.0") + ": " + (1 + upto - start) + " segments");
}
// Finally, record all merges that are viable at this level:
int end = start + m_mergeFactor;
while (end <= 1 + upto)
{
bool anyTooLarge = false;
bool anyMerging = false;
for (int i = start; i < end; i++)
{
SegmentCommitInfo info = levels[i].info;
anyTooLarge |= (Size(info) >= m_maxMergeSize || SizeDocs(info) >= m_maxMergeDocs);
if (mergingSegments.Contains(info))
{
anyMerging = true;
break;
}
}
if (anyMerging)
{
// skip
}
else if (!anyTooLarge)
{
if (spec == null)
{
spec = new MergeSpecification();
}
IList <SegmentCommitInfo> mergeInfos = new List <SegmentCommitInfo>();
for (int i = start; i < end; i++)
{
mergeInfos.Add(levels[i].info);
Debug.Assert(infos.Contains(levels[i].info));
}
if (IsVerbose)
{
Message(" add merge=" + m_writer.Get().SegString(mergeInfos) + " start=" + start + " end=" + end);
}
spec.Add(new OneMerge(mergeInfos));
}
else if (IsVerbose)
{
Message(" " + start + " to " + end + ": contains segment over maxMergeSize or maxMergeDocs; skipping");
}
start = end;
end = start + m_mergeFactor;
}
start = 1 + upto;
}
return(spec);
}
// LUCENENET TODO: Get rid of the nullable in IDictionary<SegmentCommitInfo, bool?>, if possible
/// <summary>
/// Returns the merges necessary to merge the index down
/// to a specified number of segments.
/// this respects the <see cref="m_maxMergeSizeForForcedMerge"/> setting.
/// By default, and assuming <c>maxNumSegments=1</c>, only
/// one segment will be left in the index, where that segment
/// has no deletions pending nor separate norms, and it is in
/// compound file format if the current useCompoundFile
/// setting is <c>true</c>. This method returns multiple merges
/// (mergeFactor at a time) so the <see cref="MergeScheduler"/>
/// in use may make use of concurrency.
/// </summary>
public override MergeSpecification FindForcedMerges(SegmentInfos infos, int maxNumSegments, IDictionary <SegmentCommitInfo, bool?> segmentsToMerge)
{
Debug.Assert(maxNumSegments > 0);
if (IsVerbose)
{
Message("findForcedMerges: maxNumSegs=" + maxNumSegments + " segsToMerge=" +
string.Format(J2N.Text.StringFormatter.InvariantCulture, "{0}", segmentsToMerge));
}
// If the segments are already merged (e.g. there's only 1 segment), or
// there are <maxNumSegments:.
if (IsMerged(infos, maxNumSegments, segmentsToMerge))
{
if (IsVerbose)
{
Message("already merged; skip");
}
return(null);
}
// Find the newest (rightmost) segment that needs to
// be merged (other segments may have been flushed
// since merging started):
int last = infos.Count;
while (last > 0)
{
SegmentCommitInfo info = infos.Info(--last);
if (segmentsToMerge.ContainsKey(info))
{
last++;
break;
}
}
if (last == 0)
{
if (IsVerbose)
{
Message("last == 0; skip");
}
return(null);
}
// There is only one segment already, and it is merged
if (maxNumSegments == 1 && last == 1 && IsMerged(infos, infos.Info(0)))
{
if (IsVerbose)
{
Message("already 1 seg; skip");
}
return(null);
}
// Check if there are any segments above the threshold
bool anyTooLarge = false;
for (int i = 0; i < last; i++)
{
SegmentCommitInfo info = infos.Info(i);
if (Size(info) > m_maxMergeSizeForForcedMerge || SizeDocs(info) > m_maxMergeDocs)
{
anyTooLarge = true;
break;
}
}
if (anyTooLarge)
{
return(FindForcedMergesSizeLimit(infos, maxNumSegments, last));
}
else
{
return(FindForcedMergesMaxNumSegments(infos, maxNumSegments, last));
}
}
/// <summary>
/// Finds merges necessary to force-merge all deletes from the
/// index. We simply merge adjacent segments that have
/// deletes, up to mergeFactor at a time.
/// </summary>
public override MergeSpecification FindForcedDeletesMerges(SegmentInfos segmentInfos)
{
var segments = segmentInfos.AsList();
int numSegments = segments.Count;
if (IsVerbose)
{
Message("findForcedDeleteMerges: " + numSegments + " segments");
}
var spec = new MergeSpecification();
int firstSegmentWithDeletions = -1;
IndexWriter w = m_writer.Get();
Debug.Assert(w != null);
for (int i = 0; i < numSegments; i++)
{
SegmentCommitInfo info = segmentInfos.Info(i);
int delCount = w.NumDeletedDocs(info);
if (delCount > 0)
{
if (IsVerbose)
{
Message(" segment " + info.Info.Name + " has deletions");
}
if (firstSegmentWithDeletions == -1)
{
firstSegmentWithDeletions = i;
}
else if (i - firstSegmentWithDeletions == m_mergeFactor)
{
// We've seen mergeFactor segments in a row with
// deletions, so force a merge now:
if (IsVerbose)
{
Message(" add merge " + firstSegmentWithDeletions + " to " + (i - 1) + " inclusive");
}
spec.Add(new OneMerge(segments.SubList(firstSegmentWithDeletions, i)));
firstSegmentWithDeletions = i;
}
}
else if (firstSegmentWithDeletions != -1)
{
// End of a sequence of segments with deletions, so,
// merge those past segments even if it's fewer than
// mergeFactor segments
if (IsVerbose)
{
Message(" add merge " + firstSegmentWithDeletions + " to " + (i - 1) + " inclusive");
}
spec.Add(new OneMerge(segments.SubList(firstSegmentWithDeletions, i)));
firstSegmentWithDeletions = -1;
}
}
if (firstSegmentWithDeletions != -1)
{
if (IsVerbose)
{
Message(" add merge " + firstSegmentWithDeletions + " to " + (numSegments - 1) + " inclusive");
}
spec.Add(new OneMerge(segments.SubList(firstSegmentWithDeletions, numSegments)));
}
return(spec);
}
/// <summary>
/// Create new <see cref="SegmentReader"/> sharing core from a previous
/// <see cref="SegmentReader"/> and loading new live docs from a new
/// deletes file. Used by <see cref="DirectoryReader.OpenIfChanged(DirectoryReader)"/>.
/// </summary>
internal SegmentReader(SegmentCommitInfo si, SegmentReader sr)
: this(si, sr, si.Info.Codec.LiveDocsFormat.ReadLiveDocs(si.Info.Dir, si, IOContext.READ_ONCE), si.Info.DocCount - si.DelCount)
{
}
internal virtual void SealFlushedSegment(FlushedSegment flushedSegment)
{
Debug.Assert(flushedSegment != null);
SegmentCommitInfo newSegment = flushedSegment.segmentInfo;
IndexWriter.SetDiagnostics(newSegment.Info, IndexWriter.SOURCE_FLUSH);
IOContext context = new IOContext(new FlushInfo(newSegment.Info.DocCount, newSegment.GetSizeInBytes()));
bool success = false;
try
{
if (indexWriterConfig.UseCompoundFile)
{
filesToDelete.UnionWith(IndexWriter.CreateCompoundFile(infoStream, directory, CheckAbort.NONE, newSegment.Info, context));
newSegment.Info.UseCompoundFile = true;
}
// Have codec write SegmentInfo. Must do this after
// creating CFS so that 1) .si isn't slurped into CFS,
// and 2) .si reflects useCompoundFile=true change
// above:
codec.SegmentInfoFormat.SegmentInfoWriter.Write(directory, newSegment.Info, flushedSegment.fieldInfos, context);
// TODO: ideally we would freeze newSegment here!!
// because any changes after writing the .si will be
// lost...
// Must write deleted docs after the CFS so we don't
// slurp the del file into CFS:
if (flushedSegment.liveDocs != null)
{
int delCount = flushedSegment.delCount;
Debug.Assert(delCount > 0);
if (infoStream.IsEnabled("DWPT"))
{
infoStream.Message("DWPT", "flush: write " + delCount + " deletes gen=" + flushedSegment.segmentInfo.DelGen);
}
// TODO: we should prune the segment if it's 100%
// deleted... but merge will also catch it.
// TODO: in the NRT case it'd be better to hand
// this del vector over to the
// shortly-to-be-opened SegmentReader and let it
// carry the changes; there's no reason to use
// filesystem as intermediary here.
SegmentCommitInfo info = flushedSegment.segmentInfo;
Codec codec = info.Info.Codec;
codec.LiveDocsFormat.WriteLiveDocs(flushedSegment.liveDocs, directory, info, delCount, context);
newSegment.DelCount = delCount;
newSegment.AdvanceDelGen();
}
success = true;
}
finally
{
if (!success)
{
if (infoStream.IsEnabled("DWPT"))
{
infoStream.Message("DWPT", "hit exception creating compound file for newly flushed segment " + newSegment.Info.Name);
}
}
}
}
internal virtual FlushedSegment Flush()
{
Debug.Assert(numDocsInRAM > 0);
Debug.Assert(deleteSlice.IsEmpty, "all deletes must be applied in prepareFlush");
segmentInfo.DocCount = numDocsInRAM;
SegmentWriteState flushState = new SegmentWriteState(infoStream, directory, segmentInfo, fieldInfos.Finish(), indexWriterConfig.TermIndexInterval, pendingUpdates, new IOContext(new FlushInfo(numDocsInRAM, BytesUsed)));
double startMBUsed = BytesUsed / 1024.0 / 1024.0;
// Apply delete-by-docID now (delete-byDocID only
// happens when an exception is hit processing that
// doc, eg if analyzer has some problem w/ the text):
if (pendingUpdates.docIDs.Count > 0)
{
flushState.LiveDocs = codec.LiveDocsFormat.NewLiveDocs(numDocsInRAM);
foreach (int delDocID in pendingUpdates.docIDs)
{
flushState.LiveDocs.Clear(delDocID);
}
flushState.DelCountOnFlush = pendingUpdates.docIDs.Count;
pendingUpdates.bytesUsed.AddAndGet(-pendingUpdates.docIDs.Count * BufferedUpdates.BYTES_PER_DEL_DOCID);
pendingUpdates.docIDs.Clear();
}
if (aborting)
{
if (infoStream.IsEnabled("DWPT"))
{
infoStream.Message("DWPT", "flush: skip because aborting is set");
}
return(null);
}
if (infoStream.IsEnabled("DWPT"))
{
infoStream.Message("DWPT", "flush postings as segment " + flushState.SegmentInfo.Name + " numDocs=" + numDocsInRAM);
}
bool success = false;
try
{
consumer.Flush(flushState);
pendingUpdates.terms.Clear();
segmentInfo.SetFiles(new JCG.HashSet <string>(directory.CreatedFiles));
SegmentCommitInfo segmentInfoPerCommit = new SegmentCommitInfo(segmentInfo, 0, -1L, -1L);
if (infoStream.IsEnabled("DWPT"))
{
infoStream.Message("DWPT", "new segment has " + (flushState.LiveDocs == null ? 0 : (flushState.SegmentInfo.DocCount - flushState.DelCountOnFlush)) + " deleted docs");
infoStream.Message("DWPT", "new segment has " + (flushState.FieldInfos.HasVectors ? "vectors" : "no vectors") + "; " + (flushState.FieldInfos.HasNorms ? "norms" : "no norms") + "; " + (flushState.FieldInfos.HasDocValues ? "docValues" : "no docValues") + "; " + (flushState.FieldInfos.HasProx ? "prox" : "no prox") + "; " + (flushState.FieldInfos.HasFreq ? "freqs" : "no freqs"));
infoStream.Message("DWPT", "flushedFiles=" + string.Format(J2N.Text.StringFormatter.InvariantCulture, "{0}", segmentInfoPerCommit.GetFiles()));
infoStream.Message("DWPT", "flushed codec=" + codec);
}
BufferedUpdates segmentDeletes;
if (pendingUpdates.queries.Count == 0 && pendingUpdates.numericUpdates.Count == 0 && pendingUpdates.binaryUpdates.Count == 0)
{
pendingUpdates.Clear();
segmentDeletes = null;
}
else
{
segmentDeletes = pendingUpdates;
}
if (infoStream.IsEnabled("DWPT"))
{
double newSegmentSize = segmentInfoPerCommit.GetSizeInBytes() / 1024.0 / 1024.0;
infoStream.Message("DWPT", "flushed: segment=" + segmentInfo.Name + " ramUsed=" + startMBUsed.ToString(nf) + " MB" + " newFlushedSize(includes docstores)=" + newSegmentSize.ToString(nf) + " MB" + " docs/MB=" + (flushState.SegmentInfo.DocCount / newSegmentSize).ToString(nf));
}
Debug.Assert(segmentInfo != null);
FlushedSegment fs = new FlushedSegment(segmentInfoPerCommit, flushState.FieldInfos, segmentDeletes, flushState.LiveDocs, flushState.DelCountOnFlush);
SealFlushedSegment(fs);
success = true;
return(fs);
}
finally
{
if (!success)
{
Abort(filesToDelete);
}
}
}
/// <summary>
/// Used by near real-time search </summary>
internal static DirectoryReader Open(IndexWriter writer, SegmentInfos infos, bool applyAllDeletes)
{
// IndexWriter synchronizes externally before calling
// us, which ensures infos will not change; so there's
// no need to process segments in reverse order
int numSegments = infos.Count;
IList <SegmentReader> readers = new JCG.List <SegmentReader>();
Directory dir = writer.Directory;
SegmentInfos segmentInfos = (SegmentInfos)infos.Clone();
int infosUpto = 0;
bool success = false;
try
{
for (int i = 0; i < numSegments; i++)
{
// NOTE: important that we use infos not
// segmentInfos here, so that we are passing the
// actual instance of SegmentInfoPerCommit in
// IndexWriter's segmentInfos:
SegmentCommitInfo info = infos[i];
if (Debugging.AssertsEnabled)
{
Debugging.Assert(info.Info.Dir == dir);
}
ReadersAndUpdates rld = writer.readerPool.Get(info, true);
try
{
SegmentReader reader = rld.GetReadOnlyClone(IOContext.READ);
if (reader.NumDocs > 0 || writer.KeepFullyDeletedSegments)
{
// Steal the ref:
readers.Add(reader);
infosUpto++;
}
else
{
reader.DecRef();
segmentInfos.Remove(infosUpto);
}
}
finally
{
writer.readerPool.Release(rld);
}
}
writer.IncRefDeleter(segmentInfos);
StandardDirectoryReader result = new StandardDirectoryReader(dir, readers.ToArray(), writer, segmentInfos, writer.Config.ReaderTermsIndexDivisor, applyAllDeletes);
success = true;
return(result);
}
finally
{
if (!success)
{
foreach (SegmentReader r in readers)
{
try
{
r.DecRef();
}
catch (Exception th) when(th.IsThrowable())
{
// ignore any exception that is thrown here to not mask any original
// exception.
}
}
}
}
}
public ReadersAndUpdates(IndexWriter writer, SegmentCommitInfo info)
{
this.Info = info;
this.writer = writer;
liveDocsShared = true;
}
