// Appends a new packet of buffered deletes to the stream,
// setting its generation:
public virtual long Push(FrozenBufferedUpdates packet)
{
lock (this)
{
/*
* The insert operation must be atomic. If we let threads increment the gen
* and push the packet afterwards we risk that packets are out of order.
* With DWPT this is possible if two or more flushes are racing for pushing
* updates. If the pushed packets get our of order would loose documents
* since deletes are applied to the wrong segments.
*/
packet.DelGen = NextGen_Renamed++;
Debug.Assert(packet.Any());
Debug.Assert(CheckDeleteStats());
Debug.Assert(packet.DelGen < NextGen_Renamed);
Debug.Assert(Updates.Count == 0 || Updates[Updates.Count - 1].DelGen < packet.DelGen, "Delete packets must be in order");
Updates.Add(packet);
numTerms.AddAndGet(packet.NumTermDeletes);
bytesUsed.AddAndGet(packet.BytesUsed);
if (InfoStream.IsEnabled("BD"))
{
InfoStream.Message("BD", "push deletes " + packet + " delGen=" + packet.DelGen + " packetCount=" + Updates.Count + " totBytesUsed=" + bytesUsed.Get());
}
Debug.Assert(CheckDeleteStats());
return(packet.DelGen);
}
}
private void Prune(int count)
{
lock (this)
{
if (count > 0)
{
if (infoStream.IsEnabled("BD"))
{
infoStream.Message("BD", "pruneDeletes: prune " + count + " packets; " + (updates.Count - count) + " packets remain");
}
for (int delIDX = 0; delIDX < count; delIDX++)
{
FrozenBufferedUpdates packet = updates[delIDX];
numTerms.AddAndGet(-packet.numTermDeletes);
if (Debugging.AssertsEnabled)
{
Debugging.Assert(numTerms >= 0);
}
bytesUsed.AddAndGet(-packet.bytesUsed);
if (Debugging.AssertsEnabled)
{
Debugging.Assert(bytesUsed >= 0);
}
}
updates.SubList(0, count).Clear();
}
}
}
private void Prune(int count)
{
lock (this)
{
if (count > 0)
{
if (infoStream.IsEnabled("BD"))
{
infoStream.Message("BD", "pruneDeletes: prune " + count + " packets; " + (updates.Count - count) + " packets remain");
}
for (int delIDX = 0; delIDX < count; delIDX++)
{
FrozenBufferedUpdates packet = updates[delIDX];
numTerms.AddAndGet(-packet.numTermDeletes);
if (Debugging.AssertsEnabled)
{
Debugging.Assert(numTerms >= 0);
}
bytesUsed.AddAndGet(-packet.bytesUsed);
if (Debugging.AssertsEnabled)
{
Debugging.Assert(bytesUsed >= 0);
}
}
updates.RemoveRange(0, count); // LUCENENET: Checked count parameter for correctness
}
}
}
public virtual void TestAnyChanges()
{
DocumentsWriterDeleteQueue queue = new DocumentsWriterDeleteQueue();
int size = 200 + Random.Next(500) * RANDOM_MULTIPLIER;
int termsSinceFreeze = 0;
int queriesSinceFreeze = 0;
for (int i = 0; i < size; i++)
{
Term term = new Term("id", "" + i);
if (Random.Next(10) == 0)
{
queue.AddDelete(new TermQuery(term));
queriesSinceFreeze++;
}
else
{
queue.AddDelete(term);
termsSinceFreeze++;
}
Assert.IsTrue(queue.AnyChanges());
if (Random.Next(5) == 0)
{
FrozenBufferedUpdates freezeGlobalBuffer = queue.FreezeGlobalBuffer(null);
Assert.AreEqual(termsSinceFreeze, freezeGlobalBuffer.termCount);
Assert.AreEqual(queriesSinceFreeze, ((Query[])freezeGlobalBuffer.queries.Clone()).Length);
queriesSinceFreeze = 0;
termsSinceFreeze = 0;
Assert.IsFalse(queue.AnyChanges());
}
}
}
internal FrozenBufferedUpdates FreezeGlobalBuffer(DeleteSlice callerSlice)
{
globalBufferLock.@Lock();
/*
* Here we freeze the global buffer so we need to lock it, apply all
* deletes in the queue and reset the global slice to let the GC prune the
* queue.
*/
Node currentTail = tail; // take the current tail make this local any
// Changes after this call are applied later
// and not relevant here
if (callerSlice != null)
{
// Update the callers slices so we are on the same page
callerSlice.sliceTail = currentTail;
}
try
{
if (globalSlice.sliceTail != currentTail)
{
globalSlice.sliceTail = currentTail;
globalSlice.Apply(globalBufferedUpdates, BufferedUpdates.MAX_INT32);
}
FrozenBufferedUpdates packet = new FrozenBufferedUpdates(globalBufferedUpdates, false);
globalBufferedUpdates.Clear();
return(packet);
}
finally
{
globalBufferLock.Unlock();
}
}
protected FlushTicket(FrozenBufferedUpdates frozenUpdates)
{
if (Debugging.AssertsEnabled)
{
Debugging.Assert(frozenUpdates != null);
}
this.m_frozenUpdates = frozenUpdates;
}
internal FlushedSegment(SegmentCommitInfo segmentInfo, FieldInfos fieldInfos, BufferedUpdates segmentUpdates, IMutableBits liveDocs, int delCount)
{
this.segmentInfo = segmentInfo;
this.fieldInfos = fieldInfos;
this.segmentUpdates = segmentUpdates != null && segmentUpdates.Any() ? new FrozenBufferedUpdates(segmentUpdates, true) : null;
this.liveDocs = liveDocs;
this.delCount = delCount;
}
public virtual int NumDocsInRAM => numDocsInRAM; // public for FlushPolicy
/// <summary>
/// Prepares this DWPT for flushing. this method will freeze and return the
/// <see cref="DocumentsWriterDeleteQueue"/>s global buffer and apply all pending
/// deletes to this DWPT.
/// </summary>
internal virtual FrozenBufferedUpdates PrepareFlush()
{
Debug.Assert(numDocsInRAM > 0);
FrozenBufferedUpdates globalUpdates = deleteQueue.FreezeGlobalBuffer(deleteSlice);
/* deleteSlice can possibly be null if we have hit non-aborting exceptions during indexing and never succeeded
* adding a document. */
if (deleteSlice != null)
{
// apply all deletes before we flush and release the delete slice
deleteSlice.Apply(pendingUpdates, numDocsInRAM);
Debug.Assert(deleteSlice.IsEmpty);
deleteSlice.Reset();
}
return(globalUpdates);
}
/// <summary>
/// Publishes the flushed segment, segment private deletes (if any) and its
/// associated global delete (if present) to <see cref="IndexWriter"/>. The actual
/// publishing operation is synced on IW -> BDS so that the <see cref="SegmentInfo"/>'s
/// delete generation is always <see cref="FrozenBufferedUpdates.DelGen"/> (<paramref name="globalPacket"/>) + 1
/// </summary>
protected void PublishFlushedSegment(IndexWriter indexWriter, FlushedSegment newSegment, FrozenBufferedUpdates globalPacket)
{
Debug.Assert(newSegment != null);
Debug.Assert(newSegment.segmentInfo != null);
FrozenBufferedUpdates segmentUpdates = newSegment.segmentUpdates;
//System.out.println("FLUSH: " + newSegment.segmentInfo.info.name);
if (indexWriter.infoStream.IsEnabled("DW"))
{
indexWriter.infoStream.Message("DW", "publishFlushedSegment seg-private updates=" + segmentUpdates);
}
if (segmentUpdates != null && indexWriter.infoStream.IsEnabled("DW"))
{
indexWriter.infoStream.Message("DW", "flush: push buffered seg private updates: " + segmentUpdates);
}
// now publish!
indexWriter.PublishFlushedSegment(newSegment.segmentInfo, segmentUpdates, globalPacket);
}
/// <summary>
/// Appends a new packet of buffered deletes to the stream,
/// setting its generation:
/// </summary>
public virtual long Push(FrozenBufferedUpdates packet)
{
UninterruptableMonitor.Enter(this);
try
{
/*
* The insert operation must be atomic. If we let threads increment the gen
* and push the packet afterwards we risk that packets are out of order.
* With DWPT this is possible if two or more flushes are racing for pushing
* updates. If the pushed packets get our of order would loose documents
* since deletes are applied to the wrong segments.
*/
packet.DelGen = nextGen++;
if (Debugging.AssertsEnabled)
{
Debugging.Assert(packet.Any());
Debugging.Assert(CheckDeleteStats());
Debugging.Assert(packet.DelGen < nextGen);
Debugging.Assert(updates.Count == 0 || updates[updates.Count - 1].DelGen < packet.DelGen, "Delete packets must be in order");
}
updates.Add(packet);
numTerms.AddAndGet(packet.numTermDeletes);
bytesUsed.AddAndGet(packet.bytesUsed);
if (infoStream.IsEnabled("BD"))
{
infoStream.Message("BD", "push deletes " + packet + " delGen=" + packet.DelGen + " packetCount=" + updates.Count + " totBytesUsed=" + bytesUsed);
}
if (Debugging.AssertsEnabled)
{
Debugging.Assert(CheckDeleteStats());
}
return(packet.DelGen);
}
finally
{
UninterruptableMonitor.Exit(this);
}
}
public virtual void TestPartiallyAppliedGlobalSlice()
{
DocumentsWriterDeleteQueue queue = new DocumentsWriterDeleteQueue();
System.Reflection.FieldInfo field = typeof(DocumentsWriterDeleteQueue).GetField("globalBufferLock",
BindingFlags.NonPublic | BindingFlags.GetField | BindingFlags.Instance);
ReentrantLock @lock = (ReentrantLock)field.GetValue(queue);
@lock.Lock();
var t = new ThreadAnonymousInnerClassHelper(this, queue);
t.Start();
t.Join();
@lock.Unlock();
Assert.IsTrue(queue.AnyChanges(), "changes in del queue but not in slice yet");
queue.TryApplyGlobalSlice();
Assert.IsTrue(queue.AnyChanges(), "changes in global buffer");
FrozenBufferedUpdates freezeGlobalBuffer = queue.FreezeGlobalBuffer(null);
Assert.IsTrue(freezeGlobalBuffer.Any());
Assert.AreEqual(1, freezeGlobalBuffer.termCount);
Assert.IsFalse(queue.AnyChanges(), "all changes applied");
}
internal virtual void Update(FrozenBufferedUpdates @in)
{
iterables.Add(@in.GetTermsEnumerable());
for (int queryIdx = 0; queryIdx < @in.queries.Length; queryIdx++)
{
Query query = @in.queries[queryIdx];
queries[query] = BufferedUpdates.MAX_INT32;
}
foreach (NumericDocValuesUpdate nu in @in.numericDVUpdates)
{
NumericDocValuesUpdate clone = new NumericDocValuesUpdate(nu.term, nu.field, (long?)nu.value);
clone.docIDUpto = int.MaxValue;
numericDVUpdates.Add(clone);
}
foreach (BinaryDocValuesUpdate bu in @in.binaryDVUpdates)
{
BinaryDocValuesUpdate clone = new BinaryDocValuesUpdate(bu.term, bu.field, (BytesRef)bu.value);
clone.docIDUpto = int.MaxValue;
binaryDVUpdates.Add(clone);
}
}
internal virtual void Update(FrozenBufferedUpdates @in)
{
Iterables.Add(@in.TermsIterable());
for (int queryIdx = 0; queryIdx < @in.Queries.Length; queryIdx++)
{
Query query = @in.Queries[queryIdx];
Queries[query] = BufferedUpdates.MAX_INT;
}
foreach (NumericDocValuesUpdate nu in @in.NumericDVUpdates)
{
NumericDocValuesUpdate clone = new NumericDocValuesUpdate(nu.Term, nu.Field, (long?)nu.Value);
clone.DocIDUpto = int.MaxValue;
NumericDVUpdates.Add(clone);
}
foreach (BinaryDocValuesUpdate bu in @in.BinaryDVUpdates)
{
BinaryDocValuesUpdate clone = new BinaryDocValuesUpdate(bu.Term, bu.Field, (BytesRef)bu.Value);
clone.DocIDUpto = int.MaxValue;
BinaryDVUpdates.Add(clone);
}
}
internal GlobalDeletesTicket(FrozenBufferedUpdates frozenUpdates) // LUCENENET NOTE: Made internal rather than protected because class is sealed
: base(frozenUpdates)
{
}
public IterableAnonymousClass2(FrozenBufferedUpdates outerInstance)
{
this.outerInstance = outerInstance;
}
/// <summary>
/// Resolves the buffered deleted Term/Query/docIDs, into
/// actual deleted docIDs in the liveDocs MutableBits for
/// each SegmentReader.
/// </summary>
public virtual ApplyDeletesResult ApplyDeletesAndUpdates(IndexWriter.ReaderPool readerPool, IList <SegmentCommitInfo> infos)
{
lock (this)
{
long t0 = DateTime.Now.Millisecond;
if (infos.Count == 0)
{
return(new ApplyDeletesResult(false, NextGen_Renamed++, null));
}
Debug.Assert(CheckDeleteStats());
if (!Any())
{
if (InfoStream.IsEnabled("BD"))
{
InfoStream.Message("BD", "applyDeletes: no deletes; skipping");
}
return(new ApplyDeletesResult(false, NextGen_Renamed++, null));
}
if (InfoStream.IsEnabled("BD"))
{
InfoStream.Message("BD", "applyDeletes: infos=" + infos + " packetCount=" + Updates.Count);
}
long gen = NextGen_Renamed++;
List <SegmentCommitInfo> infos2 = new List <SegmentCommitInfo>();
infos2.AddRange(infos);
infos2.Sort(sortSegInfoByDelGen);
CoalescedUpdates coalescedUpdates = null;
bool anyNewDeletes = false;
int infosIDX = infos2.Count - 1;
int delIDX = Updates.Count - 1;
IList <SegmentCommitInfo> allDeleted = null;
while (infosIDX >= 0)
{
//System.out.println("BD: cycle delIDX=" + delIDX + " infoIDX=" + infosIDX);
FrozenBufferedUpdates packet = delIDX >= 0 ? Updates[delIDX] : null;
SegmentCommitInfo info = infos2[infosIDX];
long segGen = info.BufferedDeletesGen;
if (packet != null && segGen < packet.DelGen)
{
// System.out.println(" coalesce");
if (coalescedUpdates == null)
{
coalescedUpdates = new CoalescedUpdates();
}
if (!packet.IsSegmentPrivate)
{
/*
* Only coalesce if we are NOT on a segment private del packet: the segment private del packet
* must only applied to segments with the same delGen. Yet, if a segment is already deleted
* from the SI since it had no more documents remaining after some del packets younger than
* its segPrivate packet (higher delGen) have been applied, the segPrivate packet has not been
* removed.
*/
coalescedUpdates.Update(packet);
}
delIDX--;
}
else if (packet != null && segGen == packet.DelGen)
{
Debug.Assert(packet.IsSegmentPrivate, "Packet and Segments deletegen can only match on a segment private del packet gen=" + segGen);
//System.out.println(" eq");
// Lock order: IW -> BD -> RP
Debug.Assert(readerPool.InfoIsLive(info));
ReadersAndUpdates rld = readerPool.Get(info, true);
SegmentReader reader = rld.GetReader(IOContext.READ);
int delCount = 0;
bool segAllDeletes;
try
{
DocValuesFieldUpdates.Container dvUpdates = new DocValuesFieldUpdates.Container();
if (coalescedUpdates != null)
{
//System.out.println(" del coalesced");
delCount += (int)ApplyTermDeletes(coalescedUpdates.TermsIterable(), rld, reader);
delCount += (int)ApplyQueryDeletes(coalescedUpdates.QueriesIterable(), rld, reader);
ApplyDocValuesUpdates(coalescedUpdates.NumericDVUpdates, rld, reader, dvUpdates);
ApplyDocValuesUpdates(coalescedUpdates.BinaryDVUpdates, rld, reader, dvUpdates);
}
//System.out.println(" del exact");
// Don't delete by Term here; DocumentsWriterPerThread
// already did that on flush:
delCount += (int)ApplyQueryDeletes(packet.QueriesIterable(), rld, reader);
ApplyDocValuesUpdates(Arrays.AsList(packet.NumericDVUpdates), rld, reader, dvUpdates);
ApplyDocValuesUpdates(Arrays.AsList(packet.BinaryDVUpdates), rld, reader, dvUpdates);
if (dvUpdates.Any())
{
rld.WriteFieldUpdates(info.Info.Dir, dvUpdates);
}
int fullDelCount = rld.Info.DelCount + rld.PendingDeleteCount;
Debug.Assert(fullDelCount <= rld.Info.Info.DocCount);
segAllDeletes = fullDelCount == rld.Info.Info.DocCount;
}
finally
{
rld.Release(reader);
readerPool.Release(rld);
}
anyNewDeletes |= delCount > 0;
if (segAllDeletes)
{
if (allDeleted == null)
{
allDeleted = new List <SegmentCommitInfo>();
}
allDeleted.Add(info);
}
if (InfoStream.IsEnabled("BD"))
{
InfoStream.Message("BD", "seg=" + info + " segGen=" + segGen + " segDeletes=[" + packet + "]; coalesced deletes=[" + (coalescedUpdates == null ? "null" : coalescedUpdates.ToString()) + "] newDelCount=" + delCount + (segAllDeletes ? " 100% deleted" : ""));
}
if (coalescedUpdates == null)
{
coalescedUpdates = new CoalescedUpdates();
}
/*
* Since we are on a segment private del packet we must not
* update the coalescedDeletes here! We can simply advance to the
* next packet and seginfo.
*/
delIDX--;
infosIDX--;
info.BufferedDeletesGen = gen;
}
else
{
//System.out.println(" gt");
if (coalescedUpdates != null)
{
// Lock order: IW -> BD -> RP
Debug.Assert(readerPool.InfoIsLive(info));
ReadersAndUpdates rld = readerPool.Get(info, true);
SegmentReader reader = rld.GetReader(IOContext.READ);
int delCount = 0;
bool segAllDeletes;
try
{
delCount += (int)ApplyTermDeletes(coalescedUpdates.TermsIterable(), rld, reader);
delCount += (int)ApplyQueryDeletes(coalescedUpdates.QueriesIterable(), rld, reader);
DocValuesFieldUpdates.Container dvUpdates = new DocValuesFieldUpdates.Container();
ApplyDocValuesUpdates(coalescedUpdates.NumericDVUpdates, rld, reader, dvUpdates);
ApplyDocValuesUpdates(coalescedUpdates.BinaryDVUpdates, rld, reader, dvUpdates);
if (dvUpdates.Any())
{
rld.WriteFieldUpdates(info.Info.Dir, dvUpdates);
}
int fullDelCount = rld.Info.DelCount + rld.PendingDeleteCount;
Debug.Assert(fullDelCount <= rld.Info.Info.DocCount);
segAllDeletes = fullDelCount == rld.Info.Info.DocCount;
}
finally
{
rld.Release(reader);
readerPool.Release(rld);
}
anyNewDeletes |= delCount > 0;
if (segAllDeletes)
{
if (allDeleted == null)
{
allDeleted = new List <SegmentCommitInfo>();
}
allDeleted.Add(info);
}
if (InfoStream.IsEnabled("BD"))
{
InfoStream.Message("BD", "seg=" + info + " segGen=" + segGen + " coalesced deletes=[" + coalescedUpdates + "] newDelCount=" + delCount + (segAllDeletes ? " 100% deleted" : ""));
}
}
info.BufferedDeletesGen = gen;
infosIDX--;
}
}
Debug.Assert(CheckDeleteStats());
if (InfoStream.IsEnabled("BD"))
{
InfoStream.Message("BD", "applyDeletes took " + (DateTime.Now.Millisecond - t0) + " msec");
}
// assert infos != segmentInfos || !any() : "infos=" + infos + " segmentInfos=" + segmentInfos + " any=" + any;
return(new ApplyDeletesResult(anyNewDeletes, gen, allDeleted));
}
}
protected internal SegmentFlushTicket(FrozenBufferedUpdates frozenDeletes)
: base(frozenDeletes)
{
}
public IterableAnonymousInnerClassHelper2(FrozenBufferedUpdates outerInstance)
{
this.outerInstance = outerInstance;
}
protected void FinishFlush(IndexWriter indexWriter, FlushedSegment newSegment, FrozenBufferedUpdates bufferedUpdates)
{
// Finish the flushed segment and publish it to IndexWriter
if (newSegment == null)
{
Debug.Assert(bufferedUpdates != null);
if (bufferedUpdates != null && bufferedUpdates.Any())
{
indexWriter.PublishFrozenUpdates(bufferedUpdates);
if (indexWriter.infoStream.IsEnabled("DW"))
{
indexWriter.infoStream.Message("DW", "flush: push buffered updates: " + bufferedUpdates);
}
}
}
else
{
PublishFlushedSegment(indexWriter, newSegment, bufferedUpdates);
}
}
protected FlushTicket(FrozenBufferedUpdates frozenUpdates)
{
Debug.Assert(frozenUpdates != null);
this.m_frozenUpdates = frozenUpdates;
}
internal SegmentFlushTicket(FrozenBufferedUpdates frozenDeletes) // LUCENENET NOTE: Made internal rather than protected because class is sealed
: base(frozenDeletes)
{
}
protected internal GlobalDeletesTicket(FrozenBufferedUpdates frozenUpdates)
: base(frozenUpdates)
{
}