| public UpdateSlice ( DeleteSlice slice ) : bool | ||
| slice | DeleteSlice | |
| return | bool | |
public virtual void TestUpdateDelteSlices()
{
DocumentsWriterDeleteQueue queue = new DocumentsWriterDeleteQueue();
int size = 200 + Random.Next(500) * RANDOM_MULTIPLIER;
int?[] ids = new int?[size];
for (int i = 0; i < ids.Length; i++)
{
ids[i] = Random.Next();
}
DeleteSlice slice1 = queue.NewSlice();
DeleteSlice slice2 = queue.NewSlice();
BufferedUpdates bd1 = new BufferedUpdates();
BufferedUpdates bd2 = new BufferedUpdates();
int last1 = 0;
int last2 = 0;
ISet <Term> uniqueValues = new JCG.HashSet <Term>();
for (int j = 0; j < ids.Length; j++)
{
int?i = ids[j];
// create an array here since we compare identity below against tailItem
Term[] term = new Term[] { new Term("id", i.ToString()) };
uniqueValues.Add(term[0]);
queue.AddDelete(term);
if (Random.Next(20) == 0 || j == ids.Length - 1)
{
queue.UpdateSlice(slice1);
Assert.IsTrue(slice1.IsTailItem(term));
slice1.Apply(bd1, j);
AssertAllBetween(last1, j, bd1, ids);
last1 = j + 1;
}
if (Random.Next(10) == 5 || j == ids.Length - 1)
{
queue.UpdateSlice(slice2);
Assert.IsTrue(slice2.IsTailItem(term));
slice2.Apply(bd2, j);
AssertAllBetween(last2, j, bd2, ids);
last2 = j + 1;
}
Assert.AreEqual(j + 1, queue.NumGlobalTermDeletes);
}
assertEquals(uniqueValues, new JCG.HashSet <Term>(bd1.terms.Keys));
assertEquals(uniqueValues, new JCG.HashSet <Term>(bd2.terms.Keys));
var frozenSet = new JCG.HashSet <Term>();
foreach (Term t in queue.FreezeGlobalBuffer(null).GetTermsEnumerable())
{
BytesRef bytesRef = new BytesRef();
bytesRef.CopyBytes(t.Bytes);
frozenSet.Add(new Term(t.Field, bytesRef));
}
assertEquals(uniqueValues, frozenSet);
Assert.AreEqual(0, queue.NumGlobalTermDeletes, "num deletes must be 0 after freeze");
}
public virtual void TestUpdateDelteSlices()
{
DocumentsWriterDeleteQueue queue = new DocumentsWriterDeleteQueue();
int size = 200 + Random().Next(500) * RANDOM_MULTIPLIER;
int?[] ids = new int?[size];
for (int i = 0; i < ids.Length; i++)
{
ids[i] = Random().Next();
}
DeleteSlice slice1 = queue.NewSlice();
DeleteSlice slice2 = queue.NewSlice();
BufferedUpdates bd1 = new BufferedUpdates();
BufferedUpdates bd2 = new BufferedUpdates();
int last1 = 0;
int last2 = 0;
HashSet<Term> uniqueValues = new HashSet<Term>();
for (int j = 0; j < ids.Length; j++)
{
int? i = ids[j];
// create an array here since we compare identity below against tailItem
Term[] term = new Term[] { new Term("id", i.ToString()) };
uniqueValues.Add(term[0]);
queue.AddDelete(term);
if (Random().Next(20) == 0 || j == ids.Length - 1)
{
queue.UpdateSlice(slice1);
Assert.IsTrue(slice1.IsTailItem(term));
slice1.Apply(bd1, j);
AssertAllBetween(last1, j, bd1, ids);
last1 = j + 1;
}
if (Random().Next(10) == 5 || j == ids.Length - 1)
{
queue.UpdateSlice(slice2);
Assert.IsTrue(slice2.IsTailItem(term));
slice2.Apply(bd2, j);
AssertAllBetween(last2, j, bd2, ids);
last2 = j + 1;
}
Assert.AreEqual(j + 1, queue.NumGlobalTermDeletes());
}
Assert.AreEqual(uniqueValues, bd1.Terms_Nunit().Keys);
Assert.AreEqual(uniqueValues, bd2.Terms_Nunit().Keys);
HashSet<Term> frozenSet = new HashSet<Term>();
foreach (Term t in queue.FreezeGlobalBuffer(null).TermsIterable())
{
BytesRef bytesRef = new BytesRef();
bytesRef.CopyBytes(t.Bytes());
frozenSet.Add(new Term(t.Field(), bytesRef));
}
Assert.AreEqual(uniqueValues, frozenSet);
Assert.AreEqual(0, queue.NumGlobalTermDeletes(), "num deletes must be 0 after freeze");
}
private void FinishDocument(Term delTerm)
{
/*
* here we actually finish the document in two steps 1. push the delete into
* the queue and update our slice. 2. increment the DWPT private document
* id.
*
* the updated slice we get from 1. holds all the deletes that have occurred
* since we updated the slice the last time.
*/
bool applySlice = numDocsInRAM != 0;
if (delTerm != null)
{
deleteQueue.Add(delTerm, deleteSlice);
if (Debugging.AssertsEnabled)
{
Debugging.Assert(deleteSlice.IsTailItem(delTerm), "expected the delete term as the tail item");
}
}
else
{
applySlice &= deleteQueue.UpdateSlice(deleteSlice);
}
if (applySlice)
{
deleteSlice.Apply(pendingUpdates, numDocsInRAM);
} // if we don't need to apply we must reset!
else
{
deleteSlice.Reset();
}
++numDocsInRAM;
}
public virtual void TestStressDeleteQueue()
{
DocumentsWriterDeleteQueue queue = new DocumentsWriterDeleteQueue();
ISet <Term> uniqueValues = new JCG.HashSet <Term>();
int size = 10000 + Random.Next(500) * RANDOM_MULTIPLIER;
int?[] ids = new int?[size];
for (int i = 0; i < ids.Length; i++)
{
ids[i] = Random.Next();
uniqueValues.Add(new Term("id", ids[i].ToString()));
}
CountdownEvent latch = new CountdownEvent(1);
AtomicInt32 index = new AtomicInt32(0);
int numThreads = 2 + Random.Next(5);
UpdateThread[] threads = new UpdateThread[numThreads];
for (int i = 0; i < threads.Length; i++)
{
threads[i] = new UpdateThread(queue, index, ids, latch);
threads[i].Start();
}
latch.Signal();
for (int i = 0; i < threads.Length; i++)
{
threads[i].Join();
}
foreach (UpdateThread updateThread in threads)
{
DeleteSlice slice = updateThread.Slice;
queue.UpdateSlice(slice);
BufferedUpdates deletes = updateThread.Deletes;
slice.Apply(deletes, BufferedUpdates.MAX_INT32);
assertEquals(uniqueValues, new JCG.HashSet <Term>(deletes.terms.Keys));
}
queue.TryApplyGlobalSlice();
ISet <Term> frozenSet = new JCG.HashSet <Term>();
foreach (Term t in queue.FreezeGlobalBuffer(null).GetTermsEnumerable())
{
BytesRef bytesRef = new BytesRef();
bytesRef.CopyBytes(t.Bytes);
frozenSet.Add(new Term(t.Field, bytesRef));
}
Assert.AreEqual(0, queue.NumGlobalTermDeletes, "num deletes must be 0 after freeze");
Assert.AreEqual(uniqueValues.Count, frozenSet.Count);
assertEquals(uniqueValues, frozenSet);
}
public virtual void TestStressDeleteQueue()
{
DocumentsWriterDeleteQueue queue = new DocumentsWriterDeleteQueue();
HashSet<Term> uniqueValues = new HashSet<Term>();
int size = 10000 + Random().Next(500) * RANDOM_MULTIPLIER;
int?[] ids = new int?[size];
for (int i = 0; i < ids.Length; i++)
{
ids[i] = Random().Next();
uniqueValues.Add(new Term("id", ids[i].ToString()));
}
CountDownLatch latch = new CountDownLatch(1);
AtomicInteger index = new AtomicInteger(0);
int numThreads = 2 + Random().Next(5);
UpdateThread[] threads = new UpdateThread[numThreads];
for (int i = 0; i < threads.Length; i++)
{
threads[i] = new UpdateThread(queue, index, ids, latch);
threads[i].Start();
}
latch.countDown();
for (int i = 0; i < threads.Length; i++)
{
threads[i].Join();
}
foreach (UpdateThread updateThread in threads)
{
DeleteSlice slice = updateThread.Slice;
queue.UpdateSlice(slice);
BufferedUpdates deletes = updateThread.Deletes;
slice.Apply(deletes, BufferedUpdates.MAX_INT);
Assert.AreEqual(uniqueValues, deletes.Terms_Nunit().Keys);
}
queue.TryApplyGlobalSlice();
HashSet<Term> frozenSet = new HashSet<Term>();
foreach (Term t in queue.FreezeGlobalBuffer(null).TermsIterable())
{
BytesRef bytesRef = new BytesRef();
bytesRef.CopyBytes(t.Bytes());
frozenSet.Add(new Term(t.Field(), bytesRef));
}
Assert.AreEqual(0, queue.NumGlobalTermDeletes(), "num deletes must be 0 after freeze");
Assert.AreEqual(uniqueValues.Count, frozenSet.Count);
Assert.AreEqual(uniqueValues, frozenSet);
}
