/// <summary>Checks whether this merge involves any segments
/// already participating in a merge. If not, this merge
/// is "registered", meaning we record that its segments
/// are now participating in a merge, and true is
/// returned. Else (the merge conflicts) false is
/// returned.
/// </summary>
internal bool RegisterMerge(MergePolicy.OneMerge merge)
{
lock (this)
{
if (merge.registerDone)
return true;
if (stopMerges)
{
merge.Abort();
throw new MergePolicy.MergeAbortedException("merge is aborted: " + merge.SegString(directory));
}
int count = merge.segments.Count;
bool isExternal = false;
for (int i = 0; i < count; i++)
{
SegmentInfo info = merge.segments.Info(i);
if (mergingSegments.Contains(info))
{
return false;
}
if (segmentInfos.IndexOf(info) == -1)
{
return false;
}
if (info.dir != directory)
{
isExternal = true;
}
if (segmentsToOptimize.Contains(info))
{
merge.optimize = true;
merge.maxNumSegmentsOptimize = optimizeMaxNumSegments;
}
}
EnsureContiguousMerge(merge);
pendingMerges.AddLast(merge);
if (infoStream != null)
Message("add merge to pendingMerges: " + merge.SegString(directory) + " [total " + pendingMerges.Count + " pending]");
merge.mergeGen = mergeGen;
merge.isExternal = isExternal;
// OK it does not conflict; now record that this merge
// is running (while synchronized) to avoid race
// condition where two conflicting merges from different
// threads, start
for (int i = 0; i < count; i++)
{
SegmentInfo si = merge.segments.Info(i);
mergingSegments[si] = si;
}
// Merge is now registered
merge.registerDone = true;
return true;
}
}
/// <summary>Does the actual (time-consuming) work of the merge,
/// but without holding synchronized lock on IndexWriter
/// instance
/// </summary>
private int MergeMiddle(MergePolicy.OneMerge merge)
{
merge.CheckAborted(directory);
System.String mergedName = merge.info.name;
SegmentMerger merger = null;
int mergedDocCount = 0;
SegmentInfos sourceSegments = merge.segments;
int numSegments = sourceSegments.Count;
if (infoStream != null)
Message("merging " + merge.SegString(directory));
merger = new SegmentMerger(this, mergedName, merge);
merge.readers = new SegmentReader[numSegments];
merge.readersClone = new SegmentReader[numSegments];
bool mergeDocStores = false;
System.Collections.Hashtable dss = new System.Collections.Hashtable();
String currentDocStoreSegment;
lock(this) {
currentDocStoreSegment = docWriter.GetDocStoreSegment();
}
bool currentDSSMerged = false;
// This is try/finally to make sure merger's readers are
// closed:
bool success = false;
try
{
int totDocCount = 0;
for (int i = 0; i < numSegments; i++)
{
SegmentInfo info = sourceSegments.Info(i);
// Hold onto the "live" reader; we will use this to
// commit merged deletes
SegmentReader reader = merge.readers[i] = readerPool.Get(info, merge.mergeDocStores, MERGE_READ_BUFFER_SIZE, -1);
// We clone the segment readers because other
// deletes may come in while we're merging so we
// need readers that will not change
SegmentReader clone = merge.readersClone[i] = (SegmentReader)reader.Clone(true);
merger.Add(clone);
if (clone.HasDeletions())
{
mergeDocStores = true;
}
if (info.GetDocStoreOffset() != -1 && currentDocStoreSegment != null)
{
currentDSSMerged |= currentDocStoreSegment.Equals(info.GetDocStoreSegment());
}
totDocCount += clone.NumDocs();
}
if (infoStream != null)
{
Message("merge: total " + totDocCount + " docs");
}
merge.CheckAborted(directory);
// If deletions have arrived and it has now become
// necessary to merge doc stores, go and open them:
if (mergeDocStores && !merge.mergeDocStores)
{
merge.mergeDocStores = true;
lock (this)
{
if (currentDSSMerged)
{
if (infoStream != null)
{
Message("now flush at mergeMiddle");
}
DoFlush(true, false);
}
}
for (int i = 0; i < numSegments; i++)
{
merge.readersClone[i].OpenDocStores();
}
// Clear DSS
merge.info.SetDocStore(-1, null, false);
}
// This is where all the work happens:
mergedDocCount = merge.info.docCount = merger.Merge(merge.mergeDocStores);
System.Diagnostics.Debug.Assert(mergedDocCount == totDocCount);
if (merge.useCompoundFile)
{
success = false;
string compoundFileName = IndexFileNames.SegmentFileName(mergedName, IndexFileNames.COMPOUND_FILE_EXTENSION);
try
{
if (infoStream != null)
{
Message("create compound file " + compoundFileName);
}
merger.CreateCompoundFile(compoundFileName);
success = true;
}
catch (System.IO.IOException ioe)
{
lock (this)
{
if (merge.IsAborted())
{
// This can happen if rollback or close(false)
// is called -- fall through to logic below to
// remove the partially created CFS:
}
else
{
HandleMergeException(ioe, merge);
}
}
}
catch (Exception t)
{
HandleMergeException(t, merge);
}
finally
{
if (!success)
{
if (infoStream != null)
{
Message("hit exception creating compound file during merge");
}
lock (this)
{
deleter.DeleteFile(compoundFileName);
deleter.DeleteNewFiles(merger.GetMergedFiles());
}
}
}
success = false;
lock (this)
{
// delete new non cfs files directly: they were never
// registered with IFD
deleter.DeleteNewFiles(merger.GetMergedFiles());
if (merge.IsAborted())
{
if (infoStream != null)
{
Message("abort merge after building CFS");
}
deleter.DeleteFile(compoundFileName);
return 0;
}
}
merge.info.SetUseCompoundFile(true);
}
int termsIndexDivisor;
bool loadDocStores;
// if the merged segment warmer was not installed when
// this merge was started, causing us to not force
// the docStores to close, we can't warm it now
bool canWarm = merge.info.GetDocStoreSegment() == null || currentDocStoreSegment == null || !merge.info.GetDocStoreSegment().Equals(currentDocStoreSegment);
if (poolReaders && mergedSegmentWarmer != null && canWarm)
{
// Load terms index & doc stores so the segment
// warmer can run searches, load documents/term
// vectors
termsIndexDivisor = readerTermsIndexDivisor;
loadDocStores = true;
}
else
{
termsIndexDivisor = -1;
loadDocStores = false;
}
// TODO: in the non-realtime case, we may want to only
// keep deletes (it's costly to open entire reader
// when we just need deletes)
SegmentReader mergedReader = readerPool.Get(merge.info, loadDocStores, BufferedIndexInput.BUFFER_SIZE, termsIndexDivisor);
try
{
if (poolReaders && mergedSegmentWarmer != null)
{
mergedSegmentWarmer.Warm(mergedReader);
}
if (!CommitMerge(merge, merger, mergedDocCount, mergedReader))
{
// commitMerge will return false if this merge was aborted
return 0;
}
}
finally
{
lock (this)
{
readerPool.Release(mergedReader);
}
}
success = true;
}
finally
{
// Readers are already closed in commitMerge if we didn't hit
// an exc:
if (!success)
{
CloseMergeReaders(merge, true);
}
}
merge.mergeDone = true;
lock (mergeScheduler)
{
System.Threading.Monitor.PulseAll(mergeScheduler);
}
// Force a sync after commiting the merge. Once this
// sync completes then all index files referenced by the
// current segmentInfos are on stable storage so if the
// OS/machine crashes, or power cord is yanked, the
// index will be intact. Note that this is just one
// (somewhat arbitrary) policy; we could try other
// policies like only sync if it's been > X minutes or
// more than Y bytes have been written, etc.
if (autoCommit)
{
long size;
lock (this)
{
size = merge.info.SizeInBytes();
}
Commit(size);
}
return mergedDocCount;
}
/// <summary> Merges the indicated segments, replacing them in the stack with a
/// single segment.
/// </summary>
internal void Merge(MergePolicy.OneMerge merge)
{
bool success = false;
try
{
try
{
try
{
MergeInit(merge);
if (infoStream != null)
{
Message("now merge\n merge=" + merge.SegString(directory) + "\n merge=" + merge + "\n index=" + SegString());
}
MergeMiddle(merge);
MergeSuccess(merge);
success = true;
}
catch (System.Exception t)
{
HandleMergeException(t, merge);
}
}
finally
{
lock (this)
{
MergeFinish(merge);
if (!success)
{
if (infoStream != null)
Message("hit exception during merge");
if (merge.info != null && !segmentInfos.Contains(merge.info))
deleter.Refresh(merge.info.name);
}
// This merge (and, generally, any change to the
// segments) may now enable new merges, so we call
// merge policy & update pending merges.
if (success && !merge.IsAborted() && !closed && !closing)
UpdatePendingMerges(merge.maxNumSegmentsOptimize, merge.optimize);
}
}
}
catch (System.OutOfMemoryException oom)
{
HandleOOM(oom, "merge");
}
}
private void HandleMergeException(System.Exception t, MergePolicy.OneMerge merge)
{
if (infoStream != null)
{
Message("handleMergeException: merge=" + merge.SegString(directory) + " exc=" + t);
}
// Set the exception on the merge, so if
// optimize() is waiting on us it sees the root
// cause exception:
merge.SetException(t);
AddMergeException(merge);
if (t is MergePolicy.MergeAbortedException)
{
// We can ignore this exception (it happens when
// close(false) or rollback is called), unless the
// merge involves segments from external directories,
// in which case we must throw it so, for example, the
// rollbackTransaction code in addIndexes* is
// executed.
if (merge.isExternal)
throw (MergePolicy.MergeAbortedException) t;
}
else if (t is System.IO.IOException)
throw (System.IO.IOException) t;
else if (t is System.SystemException)
throw (System.SystemException) t;
else if (t is System.ApplicationException)
throw (System.ApplicationException) t;
// Should not get here
else
throw new System.SystemException(null, t);
}
/* FIXME if we want to support non-contiguous segment merges */
private bool CommitMerge(MergePolicy.OneMerge merge, SegmentMerger merger, int mergedDocCount, SegmentReader mergedReader)
{
lock (this)
{
System.Diagnostics.Debug.Assert(TestPoint("startCommitMerge"));
if (hitOOM)
{
throw new System.SystemException("this writer hit an OutOfMemoryError; cannot complete merge");
}
if (infoStream != null)
Message("commitMerge: " + merge.SegString(directory) + " index=" + SegString());
System.Diagnostics.Debug.Assert(merge.registerDone);
// If merge was explicitly aborted, or, if rollback() or
// rollbackTransaction() had been called since our merge
// started (which results in an unqualified
// deleter.refresh() call that will remove any index
// file that current segments does not reference), we
// abort this merge
if (merge.IsAborted())
{
if (infoStream != null)
Message("commitMerge: skipping merge " + merge.SegString(directory) + ": it was aborted");
return false;
}
int start = EnsureContiguousMerge(merge);
CommitMergedDeletes(merge, mergedReader);
docWriter.RemapDeletes(segmentInfos, merger.GetDocMaps(), merger.GetDelCounts(), merge, mergedDocCount);
// If the doc store we are using has been closed and
// is in now compound format (but wasn't when we
// started), then we will switch to the compound
// format as well:
SetMergeDocStoreIsCompoundFile(merge);
merge.info.SetHasProx(merger.HasProx());
((System.Collections.IList) ((System.Collections.ArrayList) segmentInfos).GetRange(start, start + merge.segments.Count - start)).Clear();
System.Diagnostics.Debug.Assert(!segmentInfos.Contains(merge.info));
segmentInfos.Insert(start, merge.info);
CloseMergeReaders(merge, false);
// Must note the change to segmentInfos so any commits
// in-flight don't lose it:
Checkpoint();
// If the merged segments had pending changes, clear
// them so that they don't bother writing them to
// disk, updating SegmentInfo, etc.:
readerPool.Clear(merge.segments);
if (merge.optimize)
{
// cascade the optimize:
segmentsToOptimize[merge.info] = merge.info;
}
return true;
}
}
/// <summary>Carefully merges deletes for the segments we just
/// merged. This is tricky because, although merging will
/// clear all deletes (compacts the documents), new
/// deletes may have been flushed to the segments since
/// the merge was started. This method "carries over"
/// such new deletes onto the newly merged segment, and
/// saves the resulting deletes file (incrementing the
/// delete generation for merge.info). If no deletes were
/// flushed, no new deletes file is saved.
/// </summary>
private void CommitMergedDeletes(MergePolicy.OneMerge merge, SegmentReader mergeReader)
{
lock (this)
{
System.Diagnostics.Debug.Assert(TestPoint("startCommitMergeDeletes"));
SegmentInfos sourceSegments = merge.segments;
if (infoStream != null)
Message("commitMergeDeletes " + merge.SegString(directory));
// Carefully merge deletes that occurred after we
// started merging:
int docUpto = 0;
int delCount = 0;
for (int i = 0; i < sourceSegments.Count; i++)
{
SegmentInfo info = sourceSegments.Info(i);
int docCount = info.docCount;
SegmentReader previousReader = merge.readersClone[i];
SegmentReader currentReader = merge.readers[i];
if (previousReader.HasDeletions())
{
// There were deletes on this segment when the merge
// started. The merge has collapsed away those
// deletes, but, if new deletes were flushed since
// the merge started, we must now carefully keep any
// newly flushed deletes but mapping them to the new
// docIDs.
if (currentReader.NumDeletedDocs() > previousReader.NumDeletedDocs())
{
// This means this segment has had new deletes
// committed since we started the merge, so we
// must merge them:
for (int j = 0; j < docCount; j++)
{
if (previousReader.IsDeleted(j))
{
System.Diagnostics.Debug.Assert(currentReader.IsDeleted(j));
}
else
{
if (currentReader.IsDeleted(j))
{
mergeReader.DoDelete(docUpto);
delCount++;
}
docUpto++;
}
}
}
else
{
docUpto += docCount - previousReader.NumDeletedDocs();
}
}
else if (currentReader.HasDeletions())
{
// This segment had no deletes before but now it
// does:
for (int j = 0; j < docCount; j++)
{
if (currentReader.IsDeleted(j))
{
mergeReader.DoDelete(docUpto);
delCount++;
}
docUpto++;
}
}
// No deletes before or after
else
docUpto += info.docCount;
}
System.Diagnostics.Debug.Assert(mergeReader.NumDeletedDocs() == delCount);
mergeReader.hasChanges = delCount > 0;
}
}
private int EnsureContiguousMerge(MergePolicy.OneMerge merge)
{
int first = segmentInfos.IndexOf(merge.segments.Info(0));
if (first == - 1)
throw new MergePolicy.MergeException("could not find segment " + merge.segments.Info(0).name + " in current index " + SegString(), directory);
int numSegments = segmentInfos.Count;
int numSegmentsToMerge = merge.segments.Count;
for (int i = 0; i < numSegmentsToMerge; i++)
{
SegmentInfo info = merge.segments.Info(i);
if (first + i >= numSegments || !segmentInfos.Info(first + i).Equals(info))
{
if (segmentInfos.IndexOf(info) == - 1)
throw new MergePolicy.MergeException("MergePolicy selected a segment (" + info.name + ") that is not in the current index " + SegString(), directory);
else
throw new MergePolicy.MergeException("MergePolicy selected non-contiguous segments to merge (" + merge.SegString(directory) + " vs " + SegString() + "), which IndexWriter (currently) cannot handle", directory);
}
}
return first;
}