internal FrozenBufferedUpdates FreezeGlobalBuffer(DeleteSlice callerSlice)
{
globalBufferLock.@Lock();
/*
* Here we freeze the global buffer so we need to lock it, apply all
* deletes in the queue and reset the global slice to let the GC prune the
* queue.
*/
Node currentTail = tail; // take the current tail make this local any
// Changes after this call are applied later
// and not relevant here
if (callerSlice != null)
{
// Update the callers slices so we are on the same page
callerSlice.sliceTail = currentTail;
}
try
{
if (globalSlice.sliceTail != currentTail)
{
globalSlice.sliceTail = currentTail;
globalSlice.Apply(globalBufferedUpdates, BufferedUpdates.MAX_INT32);
}
FrozenBufferedUpdates packet = new FrozenBufferedUpdates(globalBufferedUpdates, false);
globalBufferedUpdates.Clear();
return(packet);
}
finally
{
globalBufferLock.Unlock();
}
}
public DocumentsWriterPerThread(string segmentName, Directory directory, LiveIndexWriterConfig indexWriterConfig, InfoStream infoStream, DocumentsWriterDeleteQueue deleteQueue, FieldInfos.Builder fieldInfos)
{
this.directoryOrig = directory;
this.directory = new TrackingDirectoryWrapper(directory);
this.fieldInfos = fieldInfos;
this.indexWriterConfig = indexWriterConfig;
this.infoStream = infoStream;
this.codec = indexWriterConfig.Codec;
this.docState = new DocState(this, infoStream);
this.docState.similarity = indexWriterConfig.Similarity;
bytesUsed = Counter.NewCounter();
byteBlockAllocator = new DirectTrackingAllocator(bytesUsed);
pendingUpdates = new BufferedUpdates();
intBlockAllocator = new Int32BlockAllocator(bytesUsed);
this.deleteQueue = deleteQueue;
if (Debugging.AssertsEnabled)
{
Debugging.Assert(numDocsInRAM == 0, "num docs {0}", numDocsInRAM);
}
pendingUpdates.Clear();
deleteSlice = deleteQueue.NewSlice();
segmentInfo = new SegmentInfo(directoryOrig, Constants.LUCENE_MAIN_VERSION, segmentName, -1, false, codec, null);
if (Debugging.AssertsEnabled)
{
Debugging.Assert(numDocsInRAM == 0);
}
if (INFO_VERBOSE && infoStream.IsEnabled("DWPT"))
{
infoStream.Message("DWPT", Thread.CurrentThread.Name + " init seg=" + segmentName + " delQueue=" + deleteQueue);
}
// this should be the last call in the ctor
// it really sucks that we need to pull this within the ctor and pass this ref to the chain!
consumer = indexWriterConfig.IndexingChain.GetChain(this);
}
internal bool UpdateSlice(DeleteSlice slice)
{
if (slice.sliceTail != tail) // If we are the same just
{
slice.sliceTail = tail;
return(true);
}
return(false);
}
protected internal UpdateThread(DocumentsWriterDeleteQueue queue, AtomicInt32 index, int?[] ids, CountdownEvent latch)
{
this.Queue = queue;
this.Index = index;
this.Ids = ids;
this.Slice = queue.NewSlice();
Deletes = new BufferedUpdates();
this.Latch = latch;
}
public bool UpdateSlice(DeleteSlice slice)
{
if (slice.SliceTail != Tail) // If we are the same just
{
slice.SliceTail = Tail;
return(true);
}
return(false);
}
public virtual void TestUpdateDelteSlices()
{
DocumentsWriterDeleteQueue queue = new DocumentsWriterDeleteQueue();
int size = 200 + Random.Next(500) * RANDOM_MULTIPLIER;
int?[] ids = new int?[size];
for (int i = 0; i < ids.Length; i++)
{
ids[i] = Random.Next();
}
DeleteSlice slice1 = queue.NewSlice();
DeleteSlice slice2 = queue.NewSlice();
BufferedUpdates bd1 = new BufferedUpdates();
BufferedUpdates bd2 = new BufferedUpdates();
int last1 = 0;
int last2 = 0;
ISet <Term> uniqueValues = new JCG.HashSet <Term>();
for (int j = 0; j < ids.Length; j++)
{
int?i = ids[j];
// create an array here since we compare identity below against tailItem
Term[] term = new Term[] { new Term("id", i.ToString()) };
uniqueValues.Add(term[0]);
queue.AddDelete(term);
if (Random.Next(20) == 0 || j == ids.Length - 1)
{
queue.UpdateSlice(slice1);
Assert.IsTrue(slice1.IsTailItem(term));
slice1.Apply(bd1, j);
AssertAllBetween(last1, j, bd1, ids);
last1 = j + 1;
}
if (Random.Next(10) == 5 || j == ids.Length - 1)
{
queue.UpdateSlice(slice2);
Assert.IsTrue(slice2.IsTailItem(term));
slice2.Apply(bd2, j);
AssertAllBetween(last2, j, bd2, ids);
last2 = j + 1;
}
Assert.AreEqual(j + 1, queue.NumGlobalTermDeletes);
}
assertEquals(uniqueValues, new JCG.HashSet <Term>(bd1.terms.Keys));
assertEquals(uniqueValues, new JCG.HashSet <Term>(bd2.terms.Keys));
var frozenSet = new JCG.HashSet <Term>();
foreach (Term t in queue.FreezeGlobalBuffer(null).GetTermsEnumerable())
{
BytesRef bytesRef = new BytesRef();
bytesRef.CopyBytes(t.Bytes);
frozenSet.Add(new Term(t.Field, bytesRef));
}
assertEquals(uniqueValues, frozenSet);
Assert.AreEqual(0, queue.NumGlobalTermDeletes, "num deletes must be 0 after freeze");
}
internal DocumentsWriterDeleteQueue(BufferedUpdates globalBufferedUpdates, long generation)
{
this.globalBufferedUpdates = globalBufferedUpdates;
this.generation = generation;
/*
* we use a sentinel instance as our initial tail. No slice will ever try to
* apply this tail since the head is always omitted.
*/
tail = new Node(null); // sentinel
globalSlice = new DeleteSlice(tail);
}
public virtual void TestStressDeleteQueue()
{
DocumentsWriterDeleteQueue queue = new DocumentsWriterDeleteQueue();
ISet <Term> uniqueValues = new JCG.HashSet <Term>();
int size = 10000 + Random.Next(500) * RANDOM_MULTIPLIER;
int?[] ids = new int?[size];
for (int i = 0; i < ids.Length; i++)
{
ids[i] = Random.Next();
uniqueValues.Add(new Term("id", ids[i].ToString()));
}
CountdownEvent latch = new CountdownEvent(1);
AtomicInt32 index = new AtomicInt32(0);
int numThreads = 2 + Random.Next(5);
UpdateThread[] threads = new UpdateThread[numThreads];
for (int i = 0; i < threads.Length; i++)
{
threads[i] = new UpdateThread(queue, index, ids, latch);
threads[i].Start();
}
latch.Signal();
for (int i = 0; i < threads.Length; i++)
{
threads[i].Join();
}
foreach (UpdateThread updateThread in threads)
{
DeleteSlice slice = updateThread.Slice;
queue.UpdateSlice(slice);
BufferedUpdates deletes = updateThread.Deletes;
slice.Apply(deletes, BufferedUpdates.MAX_INT32);
assertEquals(uniqueValues, new JCG.HashSet <Term>(deletes.terms.Keys));
}
queue.TryApplyGlobalSlice();
ISet <Term> frozenSet = new JCG.HashSet <Term>();
foreach (Term t in queue.FreezeGlobalBuffer(null).GetTermsEnumerable())
{
BytesRef bytesRef = new BytesRef();
bytesRef.CopyBytes(t.Bytes);
frozenSet.Add(new Term(t.Field, bytesRef));
}
Assert.AreEqual(0, queue.NumGlobalTermDeletes, "num deletes must be 0 after freeze");
Assert.AreEqual(uniqueValues.Count, frozenSet.Count);
assertEquals(uniqueValues, frozenSet);
}
/// <summary>
/// invariant for document update
/// </summary>
internal void Add(Term term, DeleteSlice slice)
{
TermNode termNode = new TermNode(term);
Add(termNode);
/*
* this is an update request where the term is the updated documents
* delTerm. in that case we need to guarantee that this insert is atomic
* with regards to the given delete slice. this means if two threads try to
* update the same document with in turn the same delTerm one of them must
* win. By taking the node we have created for our del term as the new tail
* it is guaranteed that if another thread adds the same right after us we
* will apply this delete next time we update our slice and one of the two
* competing updates wins!
*/
slice.sliceTail = termNode;
Debug.Assert(slice.sliceHead != slice.sliceTail, "slice head and tail must differ after add");
TryApplyGlobalSlice(); // TODO doing this each time is not necessary maybe
// we can do it just every n times or so?
}
protected internal UpdateThread(DocumentsWriterDeleteQueue queue, AtomicInteger index, int?[] ids, CountDownLatch latch)
{
this.Queue = queue;
this.Index = index;
this.Ids = ids;
this.Slice = queue.NewSlice();
Deletes = new BufferedUpdates();
this.Latch = latch;
}
public DocumentsWriterDeleteQueue(BufferedUpdates globalBufferedUpdates, long generation)
{
this.GlobalBufferedUpdates = globalBufferedUpdates;
this.Generation = generation;
/*
* we use a sentinel instance as our initial tail. No slice will ever try to
* apply this tail since the head is always omitted.
*/
Tail = new Node(null); // sentinel
GlobalSlice = new DeleteSlice(Tail);
}
public bool UpdateSlice(DeleteSlice slice)
{
if (slice.SliceTail != Tail) // If we are the same just
{
slice.SliceTail = Tail;
return true;
}
return false;
}
public FrozenBufferedUpdates FreezeGlobalBuffer(DeleteSlice callerSlice)
{
GlobalBufferLock.@Lock();
/*
* Here we freeze the global buffer so we need to lock it, apply all
* deletes in the queue and reset the global slice to let the GC prune the
* queue.
*/
Node currentTail = Tail; // take the current tail make this local any
// Changes after this call are applied later
// and not relevant here
if (callerSlice != null)
{
// Update the callers slices so we are on the same page
callerSlice.SliceTail = currentTail;
}
try
{
if (GlobalSlice.SliceTail != currentTail)
{
GlobalSlice.SliceTail = currentTail;
GlobalSlice.Apply(GlobalBufferedUpdates, BufferedUpdates.MAX_INT);
}
FrozenBufferedUpdates packet = new FrozenBufferedUpdates(GlobalBufferedUpdates, false);
GlobalBufferedUpdates.Clear();
return packet;
}
finally
{
GlobalBufferLock.Unlock();
}
}
/// <summary>
/// invariant for document update
/// </summary>
public void Add(Term term, DeleteSlice slice)
{
TermNode termNode = new TermNode(term);
Add(termNode);
/*
* this is an update request where the term is the updated documents
* delTerm. in that case we need to guarantee that this insert is atomic
* with regards to the given delete slice. this means if two threads try to
* update the same document with in turn the same delTerm one of them must
* win. By taking the node we have created for our del term as the new tail
* it is guaranteed that if another thread adds the same right after us we
* will apply this delete next time we update our slice and one of the two
* competing updates wins!
*/
slice.SliceTail = termNode;
Debug.Assert(slice.SliceHead != slice.SliceTail, "slice head and tail must differ after add");
TryApplyGlobalSlice(); // TODO doing this each time is not necessary maybe
// we can do it just every n times or so?
}
