/// <summary>
/// Carefully merges deletes and updates for the segments we just merged. this
/// is tricky because, although merging will clear all deletes (compacts the
/// documents) and compact all the updates, new deletes and updates may have
/// been flushed to the segments since the merge was started. this method
/// "carries over" such new deletes and updates onto the newly merged segment,
/// and saves the resulting deletes and updates files (incrementing the delete
/// and DV generations for merge.info). If no deletes were flushed, no new
/// deletes file is saved.
/// </summary>
private ReadersAndUpdates CommitMergedDeletesAndUpdates(MergePolicy.OneMerge merge, MergeState mergeState)
{
lock (this)
{
Debug.Assert(TestPoint("startCommitMergeDeletes"));
IList<SegmentCommitInfo> sourceSegments = merge.Segments;
if (infoStream.IsEnabled("IW"))
{
infoStream.Message("IW", "commitMergeDeletes " + SegString(merge.Segments));
}
// Carefully merge deletes that occurred after we
// started merging:
int docUpto = 0;
long minGen = long.MaxValue;
// Lazy init (only when we find a delete to carry over):
MergedDeletesAndUpdates holder = new MergedDeletesAndUpdates();
DocValuesFieldUpdates.Container mergedDVUpdates = new DocValuesFieldUpdates.Container();
for (int i = 0; i < sourceSegments.Count; i++)
{
SegmentCommitInfo info = sourceSegments[i];
minGen = Math.Min(info.BufferedDeletesGen, minGen);
int docCount = info.Info.DocCount;
Bits prevLiveDocs = merge.Readers[i].LiveDocs;
ReadersAndUpdates rld = readerPool.Get(info, false);
// We hold a ref so it should still be in the pool:
Debug.Assert(rld != null, "seg=" + info.Info.Name);
Bits currentLiveDocs = rld.LiveDocs;
IDictionary<string, DocValuesFieldUpdates> mergingFieldUpdates = rld.MergingFieldUpdates;
string[] mergingFields;
DocValuesFieldUpdates[] dvFieldUpdates;
DocValuesFieldUpdates.Iterator[] updatesIters;
if (mergingFieldUpdates.Count == 0)
{
mergingFields = null;
updatesIters = null;
dvFieldUpdates = null;
}
else
{
mergingFields = new string[mergingFieldUpdates.Count];
dvFieldUpdates = new DocValuesFieldUpdates[mergingFieldUpdates.Count];
updatesIters = new DocValuesFieldUpdates.Iterator[mergingFieldUpdates.Count];
int idx = 0;
foreach (KeyValuePair<string, DocValuesFieldUpdates> e in mergingFieldUpdates)
{
string field = e.Key;
DocValuesFieldUpdates updates = e.Value;
mergingFields[idx] = field;
dvFieldUpdates[idx] = mergedDVUpdates.GetUpdates(field, updates.Type);
if (dvFieldUpdates[idx] == null)
{
dvFieldUpdates[idx] = mergedDVUpdates.NewUpdates(field, updates.Type, mergeState.SegmentInfo.DocCount);
}
updatesIters[idx] = updates.GetIterator();
updatesIters[idx].NextDoc(); // advance to first update doc
++idx;
}
}
// System.out.println("[" + Thread.currentThread().getName() + "] IW.commitMergedDeletes: info=" + info + ", mergingUpdates=" + mergingUpdates);
if (prevLiveDocs != null)
{
// If we had deletions on starting the merge we must
// still have deletions now:
Debug.Assert(currentLiveDocs != null);
Debug.Assert(prevLiveDocs.Length() == docCount);
Debug.Assert(currentLiveDocs.Length() == docCount);
// There were deletes on this segment when the merge
// started. The merge has collapsed away those
// deletes, but, if new deletes were flushed since
// the merge started, we must now carefully keep any
// newly flushed deletes but mapping them to the new
// docIDs.
// Since we copy-on-write, if any new deletes were
// applied after merging has started, we can just
// check if the before/after liveDocs have changed.
// If so, we must carefully merge the liveDocs one
// doc at a time:
if (currentLiveDocs != prevLiveDocs)
{
// this means this segment received new deletes
// since we started the merge, so we
// must merge them:
for (int j = 0; j < docCount; j++)
{
if (!prevLiveDocs.Get(j))
{
Debug.Assert(!currentLiveDocs.Get(j));
}
else
{
if (!currentLiveDocs.Get(j))
{
if (holder.MergedDeletesAndUpdates_Renamed == null || !holder.InitializedWritableLiveDocs)
{
holder.Init(readerPool, merge, mergeState, true);
}
holder.MergedDeletesAndUpdates_Renamed.Delete(holder.DocMap.Map(docUpto));
if (mergingFields != null) // advance all iters beyond the deleted document
{
SkipDeletedDoc(updatesIters, j);
}
}
else if (mergingFields != null)
{
MaybeApplyMergedDVUpdates(merge, mergeState, docUpto, holder, mergingFields, dvFieldUpdates, updatesIters, j);
}
docUpto++;
}
}
}
else if (mergingFields != null)
{
// need to check each non-deleted document if it has any updates
for (int j = 0; j < docCount; j++)
{
if (prevLiveDocs.Get(j))
{
// document isn't deleted, check if any of the fields have an update to it
MaybeApplyMergedDVUpdates(merge, mergeState, docUpto, holder, mergingFields, dvFieldUpdates, updatesIters, j);
// advance docUpto for every non-deleted document
docUpto++;
}
else
{
// advance all iters beyond the deleted document
SkipDeletedDoc(updatesIters, j);
}
}
}
else
{
docUpto += info.Info.DocCount - info.DelCount - rld.PendingDeleteCount;
}
}
else if (currentLiveDocs != null)
{
Debug.Assert(currentLiveDocs.Length() == docCount);
// this segment had no deletes before but now it
// does:
for (int j = 0; j < docCount; j++)
{
if (!currentLiveDocs.Get(j))
{
if (holder.MergedDeletesAndUpdates_Renamed == null || !holder.InitializedWritableLiveDocs)
{
holder.Init(readerPool, merge, mergeState, true);
}
holder.MergedDeletesAndUpdates_Renamed.Delete(holder.DocMap.Map(docUpto));
if (mergingFields != null) // advance all iters beyond the deleted document
{
SkipDeletedDoc(updatesIters, j);
}
}
else if (mergingFields != null)
{
MaybeApplyMergedDVUpdates(merge, mergeState, docUpto, holder, mergingFields, dvFieldUpdates, updatesIters, j);
}
docUpto++;
}
}
else if (mergingFields != null)
{
// no deletions before or after, but there were updates
for (int j = 0; j < docCount; j++)
{
MaybeApplyMergedDVUpdates(merge, mergeState, docUpto, holder, mergingFields, dvFieldUpdates, updatesIters, j);
// advance docUpto for every non-deleted document
docUpto++;
}
}
else
{
// No deletes or updates before or after
docUpto += info.Info.DocCount;
}
}
Debug.Assert(docUpto == merge.Info_Renamed.Info.DocCount);
if (mergedDVUpdates.Any())
{
// System.out.println("[" + Thread.currentThread().getName() + "] IW.commitMergedDeletes: mergedDeletes.info=" + mergedDeletes.info + ", mergedFieldUpdates=" + mergedFieldUpdates);
bool success = false;
try
{
// if any error occurs while writing the field updates we should release
// the info, otherwise it stays in the pool but is considered not "live"
// which later causes false exceptions in pool.dropAll().
// NOTE: currently this is the only place which throws a true
// IOException. If this ever changes, we need to extend that try/finally
// block to the rest of the method too.
holder.MergedDeletesAndUpdates_Renamed.WriteFieldUpdates(directory, mergedDVUpdates);
success = true;
}
finally
{
if (!success)
{
holder.MergedDeletesAndUpdates_Renamed.DropChanges();
readerPool.Drop(merge.Info_Renamed);
}
}
}
if (infoStream.IsEnabled("IW"))
{
if (holder.MergedDeletesAndUpdates_Renamed == null)
{
infoStream.Message("IW", "no new deletes or field updates since merge started");
}
else
{
string msg = holder.MergedDeletesAndUpdates_Renamed.PendingDeleteCount + " new deletes";
if (mergedDVUpdates.Any())
{
msg += " and " + mergedDVUpdates.Size() + " new field updates";
}
msg += " since merge started";
infoStream.Message("IW", msg);
}
}
merge.Info_Renamed.BufferedDeletesGen = minGen;
return holder.MergedDeletesAndUpdates_Renamed;
}
}
private void MaybeApplyMergedDVUpdates(MergePolicy.OneMerge merge, MergeState mergeState, int docUpto, MergedDeletesAndUpdates holder, string[] mergingFields, DocValuesFieldUpdates[] dvFieldUpdates, DocValuesFieldUpdates.Iterator[] updatesIters, int curDoc)
{
int newDoc = -1;
for (int idx = 0; idx < mergingFields.Length; idx++)
{
DocValuesFieldUpdates.Iterator updatesIter = updatesIters[idx];
if (updatesIter.Doc() == curDoc) // document has an update
{
if (holder.MergedDeletesAndUpdates_Renamed == null)
{
holder.Init(readerPool, merge, mergeState, false);
}
if (newDoc == -1) // map once per all field updates, but only if there are any updates
{
newDoc = holder.DocMap.Map(docUpto);
}
DocValuesFieldUpdates dvUpdates = dvFieldUpdates[idx];
dvUpdates.Add(newDoc, updatesIter.Value());
updatesIter.NextDoc(); // advance to next document
}
else
{
Debug.Assert(updatesIter.Doc() > curDoc, "field=" + mergingFields[idx] + " updateDoc=" + updatesIter.Doc() + " curDoc=" + curDoc);
}
}
}