public virtual void TestClear()
{
DocumentsWriterDeleteQueue queue = new DocumentsWriterDeleteQueue();
Assert.IsFalse(queue.AnyChanges());
queue.Clear();
Assert.IsFalse(queue.AnyChanges());
int size = 200 + Random.Next(500) * RANDOM_MULTIPLIER;
int termsSinceFreeze = 0;
int queriesSinceFreeze = 0;
for (int i = 0; i < size; i++)
{
Term term = new Term("id", "" + i);
if (Random.Next(10) == 0)
{
queue.AddDelete(new TermQuery(term));
queriesSinceFreeze++;
}
else
{
queue.AddDelete(term);
termsSinceFreeze++;
}
Assert.IsTrue(queue.AnyChanges());
if (Random.Next(10) == 0)
{
queue.Clear();
queue.TryApplyGlobalSlice();
Assert.IsFalse(queue.AnyChanges());
}
}
}
public virtual void TestClear()
{
DocumentsWriterDeleteQueue queue = new DocumentsWriterDeleteQueue();
Assert.IsFalse(queue.AnyChanges());
queue.Clear();
Assert.IsFalse(queue.AnyChanges());
int size = 200 + Random().Next(500) * RANDOM_MULTIPLIER;
int termsSinceFreeze = 0;
int queriesSinceFreeze = 0;
for (int i = 0; i < size; i++)
{
Term term = new Term("id", "" + i);
if (Random().Next(10) == 0)
{
queue.AddDelete(new TermQuery(term));
queriesSinceFreeze++;
}
else
{
queue.AddDelete(term);
termsSinceFreeze++;
}
Assert.IsTrue(queue.AnyChanges());
if (Random().Next(10) == 0)
{
queue.Clear();
queue.TryApplyGlobalSlice();
Assert.IsFalse(queue.AnyChanges());
}
}
}
public virtual void TestAnyChanges()
{
DocumentsWriterDeleteQueue queue = new DocumentsWriterDeleteQueue();
int size = 200 + Random.Next(500) * RANDOM_MULTIPLIER;
int termsSinceFreeze = 0;
int queriesSinceFreeze = 0;
for (int i = 0; i < size; i++)
{
Term term = new Term("id", "" + i);
if (Random.Next(10) == 0)
{
queue.AddDelete(new TermQuery(term));
queriesSinceFreeze++;
}
else
{
queue.AddDelete(term);
termsSinceFreeze++;
}
Assert.IsTrue(queue.AnyChanges());
if (Random.Next(5) == 0)
{
FrozenBufferedUpdates freezeGlobalBuffer = queue.FreezeGlobalBuffer(null);
Assert.AreEqual(termsSinceFreeze, freezeGlobalBuffer.termCount);
Assert.AreEqual(queriesSinceFreeze, ((Query[])freezeGlobalBuffer.queries.Clone()).Length);
queriesSinceFreeze = 0;
termsSinceFreeze = 0;
Assert.IsFalse(queue.AnyChanges());
}
}
}
internal virtual void AddDeletes(DocumentsWriterDeleteQueue deleteQueue)
{
UninterruptableMonitor.Enter(this);
try
{
IncTickets(); // first inc the ticket count - freeze opens
// a window for #anyChanges to fail
bool success = false;
try
{
queue.Enqueue(new GlobalDeletesTicket(deleteQueue.FreezeGlobalBuffer(null)));
success = true;
}
finally
{
if (!success)
{
DecTickets();
}
}
}
finally
{
UninterruptableMonitor.Exit(this);
}
}
public virtual void TestAnyChanges()
{
DocumentsWriterDeleteQueue queue = new DocumentsWriterDeleteQueue();
int size = 200 + Random().Next(500) * RANDOM_MULTIPLIER;
int termsSinceFreeze = 0;
int queriesSinceFreeze = 0;
for (int i = 0; i < size; i++)
{
Term term = new Term("id", "" + i);
if (Random().Next(10) == 0)
{
queue.AddDelete(new TermQuery(term));
queriesSinceFreeze++;
}
else
{
queue.AddDelete(term);
termsSinceFreeze++;
}
Assert.IsTrue(queue.AnyChanges());
if (Random().Next(5) == 0)
{
FrozenBufferedUpdates freezeGlobalBuffer = queue.FreezeGlobalBuffer(null);
Assert.AreEqual(termsSinceFreeze, freezeGlobalBuffer.TermCount);
Assert.AreEqual(queriesSinceFreeze, ((Query[])freezeGlobalBuffer.Queries_Nunit()).Length);
queriesSinceFreeze = 0;
termsSinceFreeze = 0;
Assert.IsFalse(queue.AnyChanges());
}
}
}
public DocumentsWriterPerThread(string segmentName, Directory directory, LiveIndexWriterConfig indexWriterConfig, InfoStream infoStream, DocumentsWriterDeleteQueue deleteQueue, FieldInfos.Builder fieldInfos)
{
this.directoryOrig = directory;
this.directory = new TrackingDirectoryWrapper(directory);
this.fieldInfos = fieldInfos;
this.indexWriterConfig = indexWriterConfig;
this.infoStream = infoStream;
this.codec = indexWriterConfig.Codec;
this.docState = new DocState(this, infoStream);
this.docState.similarity = indexWriterConfig.Similarity;
bytesUsed = Counter.NewCounter();
byteBlockAllocator = new DirectTrackingAllocator(bytesUsed);
pendingUpdates = new BufferedUpdates();
intBlockAllocator = new Int32BlockAllocator(bytesUsed);
this.deleteQueue = deleteQueue;
if (Debugging.AssertsEnabled)
{
Debugging.Assert(numDocsInRAM == 0, "num docs {0}", numDocsInRAM);
}
pendingUpdates.Clear();
deleteSlice = deleteQueue.NewSlice();
segmentInfo = new SegmentInfo(directoryOrig, Constants.LUCENE_MAIN_VERSION, segmentName, -1, false, codec, null);
if (Debugging.AssertsEnabled)
{
Debugging.Assert(numDocsInRAM == 0);
}
if (INFO_VERBOSE && infoStream.IsEnabled("DWPT"))
{
infoStream.Message("DWPT", Thread.CurrentThread.Name + " init seg=" + segmentName + " delQueue=" + deleteQueue);
}
// this should be the last call in the ctor
// it really sucks that we need to pull this within the ctor and pass this ref to the chain!
consumer = indexWriterConfig.IndexingChain.GetChain(this);
}
// for asserts
private bool SetFlushingDeleteQueue(DocumentsWriterDeleteQueue session)
{
lock (this)
{
currentFullFlushDelQueue = session;
return(true);
}
}
internal bool AssertBlockedFlushes(DocumentsWriterDeleteQueue flushingQueue)
{
foreach (BlockedFlush blockedFlush in blockedFlushes)
{
Debug.Assert(blockedFlush.Dwpt.deleteQueue == flushingQueue);
}
return(true);
}
protected internal UpdateThread(DocumentsWriterDeleteQueue queue, AtomicInt32 index, int?[] ids, CountdownEvent latch)
{
this.Queue = queue;
this.Index = index;
this.Ids = ids;
this.Slice = queue.NewSlice();
Deletes = new BufferedUpdates();
this.Latch = latch;
}
public virtual void TestUpdateDelteSlices()
{
DocumentsWriterDeleteQueue queue = new DocumentsWriterDeleteQueue();
int size = 200 + Random.Next(500) * RANDOM_MULTIPLIER;
int?[] ids = new int?[size];
for (int i = 0; i < ids.Length; i++)
{
ids[i] = Random.Next();
}
DeleteSlice slice1 = queue.NewSlice();
DeleteSlice slice2 = queue.NewSlice();
BufferedUpdates bd1 = new BufferedUpdates();
BufferedUpdates bd2 = new BufferedUpdates();
int last1 = 0;
int last2 = 0;
ISet <Term> uniqueValues = new JCG.HashSet <Term>();
for (int j = 0; j < ids.Length; j++)
{
int?i = ids[j];
// create an array here since we compare identity below against tailItem
Term[] term = new Term[] { new Term("id", i.ToString()) };
uniqueValues.Add(term[0]);
queue.AddDelete(term);
if (Random.Next(20) == 0 || j == ids.Length - 1)
{
queue.UpdateSlice(slice1);
Assert.IsTrue(slice1.IsTailItem(term));
slice1.Apply(bd1, j);
AssertAllBetween(last1, j, bd1, ids);
last1 = j + 1;
}
if (Random.Next(10) == 5 || j == ids.Length - 1)
{
queue.UpdateSlice(slice2);
Assert.IsTrue(slice2.IsTailItem(term));
slice2.Apply(bd2, j);
AssertAllBetween(last2, j, bd2, ids);
last2 = j + 1;
}
Assert.AreEqual(j + 1, queue.NumGlobalTermDeletes);
}
assertEquals(uniqueValues, new JCG.HashSet <Term>(bd1.terms.Keys));
assertEquals(uniqueValues, new JCG.HashSet <Term>(bd2.terms.Keys));
var frozenSet = new JCG.HashSet <Term>();
foreach (Term t in queue.FreezeGlobalBuffer(null).GetTermsEnumerable())
{
BytesRef bytesRef = new BytesRef();
bytesRef.CopyBytes(t.Bytes);
frozenSet.Add(new Term(t.Field, bytesRef));
}
assertEquals(uniqueValues, frozenSet);
Assert.AreEqual(0, queue.NumGlobalTermDeletes, "num deletes must be 0 after freeze");
}
internal bool UpdateNumericDocValue(Term term, string field, long?value)
{
lock (this)
{
DocumentsWriterDeleteQueue deleteQueue = this.deleteQueue;
deleteQueue.AddNumericUpdate(new NumericDocValuesUpdate(term, field, value));
flushControl.DoOnDelete();
return(ApplyAllDeletes(deleteQueue));
}
}
internal bool UpdateBinaryDocValue(Term term, string field, BytesRef value)
{
lock (this)
{
DocumentsWriterDeleteQueue deleteQueue = this.deleteQueue;
deleteQueue.AddBinaryUpdate(new BinaryDocValuesUpdate(term, field, value));
flushControl.DoOnDelete();
return(ApplyAllDeletes(deleteQueue));
}
}
internal bool DeleteQueries(params Query[] queries)
{
lock (this)
{
// TODO why is this synchronized?
DocumentsWriterDeleteQueue deleteQueue = this.DeleteQueue;
deleteQueue.AddDelete(queries);
FlushControl.DoOnDelete();
return(ApplyAllDeletes(deleteQueue));
}
}
// TODO: we could check w/ FreqProxTermsWriter: if the
// term doesn't exist, don't bother buffering into the
// per-DWPT map (but still must go into the global map)
internal bool DeleteTerms(params Term[] terms)
{
lock (this)
{
// TODO why is this synchronized?
DocumentsWriterDeleteQueue deleteQueue = this.deleteQueue;
deleteQueue.AddDelete(terms);
flushControl.DoOnDelete();
return(ApplyAllDeletes(deleteQueue));
}
}
public virtual void TestUpdateDelteSlices()
{
DocumentsWriterDeleteQueue queue = new DocumentsWriterDeleteQueue();
int size = 200 + Random().Next(500) * RANDOM_MULTIPLIER;
int?[] ids = new int?[size];
for (int i = 0; i < ids.Length; i++)
{
ids[i] = Random().Next();
}
DeleteSlice slice1 = queue.NewSlice();
DeleteSlice slice2 = queue.NewSlice();
BufferedUpdates bd1 = new BufferedUpdates();
BufferedUpdates bd2 = new BufferedUpdates();
int last1 = 0;
int last2 = 0;
HashSet<Term> uniqueValues = new HashSet<Term>();
for (int j = 0; j < ids.Length; j++)
{
int? i = ids[j];
// create an array here since we compare identity below against tailItem
Term[] term = new Term[] { new Term("id", i.ToString()) };
uniqueValues.Add(term[0]);
queue.AddDelete(term);
if (Random().Next(20) == 0 || j == ids.Length - 1)
{
queue.UpdateSlice(slice1);
Assert.IsTrue(slice1.IsTailItem(term));
slice1.Apply(bd1, j);
AssertAllBetween(last1, j, bd1, ids);
last1 = j + 1;
}
if (Random().Next(10) == 5 || j == ids.Length - 1)
{
queue.UpdateSlice(slice2);
Assert.IsTrue(slice2.IsTailItem(term));
slice2.Apply(bd2, j);
AssertAllBetween(last2, j, bd2, ids);
last2 = j + 1;
}
Assert.AreEqual(j + 1, queue.NumGlobalTermDeletes());
}
Assert.AreEqual(uniqueValues, bd1.Terms_Nunit().Keys);
Assert.AreEqual(uniqueValues, bd2.Terms_Nunit().Keys);
HashSet<Term> frozenSet = new HashSet<Term>();
foreach (Term t in queue.FreezeGlobalBuffer(null).TermsIterable())
{
BytesRef bytesRef = new BytesRef();
bytesRef.CopyBytes(t.Bytes());
frozenSet.Add(new Term(t.Field(), bytesRef));
}
Assert.AreEqual(uniqueValues, frozenSet);
Assert.AreEqual(0, queue.NumGlobalTermDeletes(), "num deletes must be 0 after freeze");
}
// for asserts
private bool SetFlushingDeleteQueue(DocumentsWriterDeleteQueue session)
{
UninterruptableMonitor.Enter(this);
try
{
currentFullFlushDelQueue = session;
return(true);
}
finally
{
UninterruptableMonitor.Exit(this);
}
}
private bool ApplyAllDeletes(DocumentsWriterDeleteQueue deleteQueue)
{
if (flushControl.GetAndResetApplyAllDeletes())
{
if (deleteQueue != null && !flushControl.IsFullFlush)
{
ticketQueue.AddDeletes(deleteQueue);
}
PutEvent(ApplyDeletesEvent.INSTANCE); // apply deletes event forces a purge
return(true);
}
return(false);
}
public virtual void TestStressDeleteQueue()
{
DocumentsWriterDeleteQueue queue = new DocumentsWriterDeleteQueue();
ISet <Term> uniqueValues = new JCG.HashSet <Term>();
int size = 10000 + Random.Next(500) * RANDOM_MULTIPLIER;
int?[] ids = new int?[size];
for (int i = 0; i < ids.Length; i++)
{
ids[i] = Random.Next();
uniqueValues.Add(new Term("id", ids[i].ToString()));
}
CountdownEvent latch = new CountdownEvent(1);
AtomicInt32 index = new AtomicInt32(0);
int numThreads = 2 + Random.Next(5);
UpdateThread[] threads = new UpdateThread[numThreads];
for (int i = 0; i < threads.Length; i++)
{
threads[i] = new UpdateThread(queue, index, ids, latch);
threads[i].Start();
}
latch.Signal();
for (int i = 0; i < threads.Length; i++)
{
threads[i].Join();
}
foreach (UpdateThread updateThread in threads)
{
DeleteSlice slice = updateThread.Slice;
queue.UpdateSlice(slice);
BufferedUpdates deletes = updateThread.Deletes;
slice.Apply(deletes, BufferedUpdates.MAX_INT32);
assertEquals(uniqueValues, new JCG.HashSet <Term>(deletes.terms.Keys));
}
queue.TryApplyGlobalSlice();
ISet <Term> frozenSet = new JCG.HashSet <Term>();
foreach (Term t in queue.FreezeGlobalBuffer(null).GetTermsEnumerable())
{
BytesRef bytesRef = new BytesRef();
bytesRef.CopyBytes(t.Bytes);
frozenSet.Add(new Term(t.Field, bytesRef));
}
Assert.AreEqual(0, queue.NumGlobalTermDeletes, "num deletes must be 0 after freeze");
Assert.AreEqual(uniqueValues.Count, frozenSet.Count);
assertEquals(uniqueValues, frozenSet);
}
internal bool UpdateBinaryDocValue(Term term, string field, BytesRef value)
{
UninterruptableMonitor.Enter(this);
try
{
DocumentsWriterDeleteQueue deleteQueue = this.deleteQueue;
deleteQueue.AddBinaryUpdate(new BinaryDocValuesUpdate(term, field, value));
flushControl.DoOnDelete();
return(ApplyAllDeletes(deleteQueue));
}
finally
{
UninterruptableMonitor.Exit(this);
}
}
// TODO: we could check w/ FreqProxTermsWriter: if the
// term doesn't exist, don't bother buffering into the
// per-DWPT map (but still must go into the global map)
internal bool DeleteTerms(params Term[] terms)
{
UninterruptableMonitor.Enter(this);
try
{
// TODO why is this synchronized?
DocumentsWriterDeleteQueue deleteQueue = this.deleteQueue;
deleteQueue.AddDelete(terms);
flushControl.DoOnDelete();
return(ApplyAllDeletes(deleteQueue));
}
finally
{
UninterruptableMonitor.Exit(this);
}
}
private bool AssertActiveDeleteQueue(DocumentsWriterDeleteQueue queue)
{
int limit = perThreadPool.NumThreadStatesActive;
for (int i = 0; i < limit; i++)
{
ThreadState next = perThreadPool.GetThreadState(i);
next.@Lock();
try
{
Debug.Assert(!next.IsInitialized || next.dwpt.deleteQueue == queue, "isInitialized: " + next.IsInitialized + " numDocs: " + (next.IsInitialized ? next.dwpt.NumDocsInRAM : 0));
}
finally
{
next.Unlock();
}
}
return(true);
}
/// <summary>
/// Prunes the blockedQueue by removing all DWPT that are associated with the given flush queue.
/// </summary>
private void PruneBlockedQueue(DocumentsWriterDeleteQueue flushingQueue)
{
var node = blockedFlushes.First;
while (node != null)
{
var nextNode = node.Next;
BlockedFlush blockedFlush = node.Value;
if (blockedFlush.Dwpt.deleteQueue == flushingQueue)
{
blockedFlushes.Remove(node);
Debug.Assert(!flushingWriters.ContainsKey(blockedFlush.Dwpt), "DWPT is already flushing");
// Record the flushing DWPT to reduce flushBytes in doAfterFlush
flushingWriters[blockedFlush.Dwpt] = Convert.ToInt64(blockedFlush.Bytes);
// don't decr pending here - its already done when DWPT is blocked
flushQueue.Enqueue(blockedFlush.Dwpt);
}
node = nextNode;
}
}
internal virtual void AddDeletes(DocumentsWriterDeleteQueue deleteQueue)
{
lock (this)
{
IncTickets(); // first inc the ticket count - freeze opens
// a window for #anyChanges to fail
bool success = false;
try
{
Queue.AddLast(new GlobalDeletesTicket(deleteQueue.FreezeGlobalBuffer(null)));
success = true;
}
finally
{
if (!success)
{
DecTickets();
}
}
}
}
internal virtual void AddDeletes(DocumentsWriterDeleteQueue deleteQueue)
{
lock (this)
{
IncTickets(); // first inc the ticket count - freeze opens
// a window for #anyChanges to fail
bool success = false;
try
{
Queue.AddLast(new GlobalDeletesTicket(deleteQueue.FreezeGlobalBuffer(null)));
success = true;
}
finally
{
if (!success)
{
DecTickets();
}
}
}
}
/// <summary>
/// Prunes the blockedQueue by removing all DWPT that are associated with the given flush queue.
/// </summary>
private void PruneBlockedQueue(DocumentsWriterDeleteQueue flushingQueue)
{
lock (this)
{
IEnumerator <BlockedFlush> iterator = BlockedFlushes.GetEnumerator();
while (iterator.MoveNext())
{
BlockedFlush blockedFlush = iterator.Current;
if (blockedFlush.Dwpt.DeleteQueue == flushingQueue)
{
//LUCENE TODO: Move to try finally
BlockedFlushes.Remove(blockedFlush);
Debug.Assert(!FlushingWriters.ContainsKey(blockedFlush.Dwpt), "DWPT is already flushing");
// Record the flushing DWPT to reduce flushBytes in doAfterFlush
FlushingWriters[blockedFlush.Dwpt] = Convert.ToInt64(blockedFlush.Bytes);
// don't decr pending here - its already done when DWPT is blocked
FlushQueue.Enqueue(blockedFlush.Dwpt);
}
}
}
}
public virtual void TestPartiallyAppliedGlobalSlice()
{
DocumentsWriterDeleteQueue queue = new DocumentsWriterDeleteQueue();
System.Reflection.FieldInfo field = typeof(DocumentsWriterDeleteQueue).GetField("globalBufferLock",
BindingFlags.NonPublic | BindingFlags.GetField | BindingFlags.Instance);
ReentrantLock @lock = (ReentrantLock)field.GetValue(queue);
@lock.Lock();
var t = new ThreadAnonymousInnerClassHelper(this, queue);
t.Start();
t.Join();
@lock.Unlock();
Assert.IsTrue(queue.AnyChanges(), "changes in del queue but not in slice yet");
queue.TryApplyGlobalSlice();
Assert.IsTrue(queue.AnyChanges(), "changes in global buffer");
FrozenBufferedUpdates freezeGlobalBuffer = queue.FreezeGlobalBuffer(null);
Assert.IsTrue(freezeGlobalBuffer.Any());
Assert.AreEqual(1, freezeGlobalBuffer.termCount);
Assert.IsFalse(queue.AnyChanges(), "all changes applied");
}
// for asserts
private bool SetFlushingDeleteQueue(DocumentsWriterDeleteQueue session)
{
lock (this)
{
CurrentFullFlushDelQueue = session;
return true;
}
}
internal void MarkForFullFlush()
{
DocumentsWriterDeleteQueue flushingQueue;
lock (this)
{
Debug.Assert(!fullFlush, "called DWFC#markForFullFlush() while full flush is still running");
Debug.Assert(fullFlushBuffer.Count == 0, "full flush buffer should be empty: " + fullFlushBuffer);
fullFlush = true;
flushingQueue = documentsWriter.deleteQueue;
// Set a new delete queue - all subsequent DWPT will use this queue until
// we do another full flush
DocumentsWriterDeleteQueue newQueue = new DocumentsWriterDeleteQueue(flushingQueue.generation + 1);
documentsWriter.deleteQueue = newQueue;
}
int limit = perThreadPool.NumThreadStatesActive;
for (int i = 0; i < limit; i++)
{
ThreadState next = perThreadPool.GetThreadState(i);
next.@Lock();
try
{
if (!next.IsInitialized)
{
if (closed && next.IsActive)
{
perThreadPool.DeactivateThreadState(next);
}
continue;
}
Debug.Assert(next.dwpt.deleteQueue == flushingQueue || next.dwpt.deleteQueue == documentsWriter.deleteQueue, " flushingQueue: " + flushingQueue + " currentqueue: " + documentsWriter.deleteQueue + " perThread queue: " + next.dwpt.deleteQueue + " numDocsInRam: " + next.dwpt.NumDocsInRAM);
if (next.dwpt.deleteQueue != flushingQueue)
{
// this one is already a new DWPT
continue;
}
AddFlushableState(next);
}
finally
{
next.Unlock();
}
}
lock (this)
{
/* make sure we move all DWPT that are where concurrently marked as
* pending and moved to blocked are moved over to the flushQueue. There is
* a chance that this happens since we marking DWPT for full flush without
* blocking indexing.*/
PruneBlockedQueue(flushingQueue);
Debug.Assert(AssertBlockedFlushes(documentsWriter.deleteQueue));
//FlushQueue.AddAll(FullFlushBuffer);
foreach (var dwpt in fullFlushBuffer)
{
flushQueue.Enqueue(dwpt);
}
fullFlushBuffer.Clear();
UpdateStallState();
}
Debug.Assert(AssertActiveDeleteQueue(documentsWriter.deleteQueue));
}
public virtual void TestPartiallyAppliedGlobalSlice()
{
DocumentsWriterDeleteQueue queue = new DocumentsWriterDeleteQueue();
System.Reflection.FieldInfo field = typeof(DocumentsWriterDeleteQueue).GetField("GlobalBufferLock",
BindingFlags.NonPublic | BindingFlags.GetField | BindingFlags.Instance);
ReentrantLock @lock = (ReentrantLock)field.GetValue(queue);
@lock.Lock();
ThreadClass t = new ThreadAnonymousInnerClassHelper(this, queue);
t.Start();
t.Join();
@lock.Unlock();
Assert.IsTrue(queue.AnyChanges(), "changes in del queue but not in slice yet");
queue.TryApplyGlobalSlice();
Assert.IsTrue(queue.AnyChanges(), "changes in global buffer");
FrozenBufferedUpdates freezeGlobalBuffer = queue.FreezeGlobalBuffer(null);
Assert.IsTrue(freezeGlobalBuffer.Any());
Assert.AreEqual(1, freezeGlobalBuffer.TermCount);
Assert.IsFalse(queue.AnyChanges(), "all changes applied");
}
public virtual void TestStressDeleteQueue()
{
DocumentsWriterDeleteQueue queue = new DocumentsWriterDeleteQueue();
HashSet<Term> uniqueValues = new HashSet<Term>();
int size = 10000 + Random().Next(500) * RANDOM_MULTIPLIER;
int?[] ids = new int?[size];
for (int i = 0; i < ids.Length; i++)
{
ids[i] = Random().Next();
uniqueValues.Add(new Term("id", ids[i].ToString()));
}
CountDownLatch latch = new CountDownLatch(1);
AtomicInteger index = new AtomicInteger(0);
int numThreads = 2 + Random().Next(5);
UpdateThread[] threads = new UpdateThread[numThreads];
for (int i = 0; i < threads.Length; i++)
{
threads[i] = new UpdateThread(queue, index, ids, latch);
threads[i].Start();
}
latch.countDown();
for (int i = 0; i < threads.Length; i++)
{
threads[i].Join();
}
foreach (UpdateThread updateThread in threads)
{
DeleteSlice slice = updateThread.Slice;
queue.UpdateSlice(slice);
BufferedUpdates deletes = updateThread.Deletes;
slice.Apply(deletes, BufferedUpdates.MAX_INT);
Assert.AreEqual(uniqueValues, deletes.Terms_Nunit().Keys);
}
queue.TryApplyGlobalSlice();
HashSet<Term> frozenSet = new HashSet<Term>();
foreach (Term t in queue.FreezeGlobalBuffer(null).TermsIterable())
{
BytesRef bytesRef = new BytesRef();
bytesRef.CopyBytes(t.Bytes());
frozenSet.Add(new Term(t.Field(), bytesRef));
}
Assert.AreEqual(0, queue.NumGlobalTermDeletes(), "num deletes must be 0 after freeze");
Assert.AreEqual(uniqueValues.Count, frozenSet.Count);
Assert.AreEqual(uniqueValues, frozenSet);
}
private bool ApplyAllDeletes(DocumentsWriterDeleteQueue deleteQueue)
{
if (FlushControl.AndResetApplyAllDeletes)
{
if (deleteQueue != null && !FlushControl.FullFlush)
{
TicketQueue.AddDeletes(deleteQueue);
}
PutEvent(ApplyDeletesEvent.INSTANCE); // apply deletes event forces a purge
return true;
}
return false;
}
public ThreadAnonymousInnerClassHelper(TestDocumentsWriterDeleteQueue outerInstance, DocumentsWriterDeleteQueue queue)
{
this.OuterInstance = outerInstance;
this.Queue = queue;
}
public DocumentsWriterPerThread(string segmentName, Directory directory, LiveIndexWriterConfig indexWriterConfig, InfoStream infoStream, DocumentsWriterDeleteQueue deleteQueue, FieldInfos.Builder fieldInfos)
{
this.DirectoryOrig = directory;
this.Directory = new TrackingDirectoryWrapper(directory);
this.FieldInfos = fieldInfos;
this.IndexWriterConfig = indexWriterConfig;
this.InfoStream = infoStream;
this.Codec = indexWriterConfig.Codec;
this.docState = new DocState(this, infoStream);
this.docState.Similarity = indexWriterConfig.Similarity;
bytesUsed = Counter.NewCounter();
ByteBlockAllocator = new DirectTrackingAllocator(bytesUsed);
PendingUpdates = new BufferedUpdates();
intBlockAllocator = new IntBlockAllocator(bytesUsed);
this.DeleteQueue = deleteQueue;
Debug.Assert(numDocsInRAM == 0, "num docs " + numDocsInRAM);
PendingUpdates.Clear();
DeleteSlice = deleteQueue.NewSlice();
SegmentInfo_Renamed = new SegmentInfo(DirectoryOrig, Constants.LUCENE_MAIN_VERSION, segmentName, -1, false, Codec, null);
Debug.Assert(numDocsInRAM == 0);
if (INFO_VERBOSE && infoStream.IsEnabled("DWPT"))
{
infoStream.Message("DWPT", Thread.CurrentThread.Name + " init seg=" + segmentName + " delQueue=" + deleteQueue);
}
// this should be the last call in the ctor
// it really sucks that we need to pull this within the ctor and pass this ref to the chain!
Consumer = indexWriterConfig.IndexingChain.GetChain(this);
}
public ThreadAnonymousInnerClassHelper(TestDocumentsWriterDeleteQueue outerInstance, DocumentsWriterDeleteQueue queue)
{
this.OuterInstance = outerInstance;
this.Queue = queue;
}
public ThreadAnonymousClass(TestDocumentsWriterDeleteQueue outerInstance, DocumentsWriterDeleteQueue queue)
{
this.outerInstance = outerInstance;
this.queue = queue;
}
protected internal UpdateThread(DocumentsWriterDeleteQueue queue, AtomicInteger index, int?[] ids, CountDownLatch latch)
{
this.Queue = queue;
this.Index = index;
this.Ids = ids;
this.Slice = queue.NewSlice();
Deletes = new BufferedUpdates();
this.Latch = latch;
}