public virtual void TestUpdateDelteSlices()
{
DocumentsWriterDeleteQueue queue = new DocumentsWriterDeleteQueue();
int size = 200 + Random().Next(500) * RANDOM_MULTIPLIER;
int?[] ids = new int?[size];
for (int i = 0; i < ids.Length; i++)
{
ids[i] = Random().Next();
}
DeleteSlice slice1 = queue.NewSlice();
DeleteSlice slice2 = queue.NewSlice();
BufferedUpdates bd1 = new BufferedUpdates();
BufferedUpdates bd2 = new BufferedUpdates();
int last1 = 0;
int last2 = 0;
HashSet<Term> uniqueValues = new HashSet<Term>();
for (int j = 0; j < ids.Length; j++)
{
int? i = ids[j];
// create an array here since we compare identity below against tailItem
Term[] term = new Term[] { new Term("id", i.ToString()) };
uniqueValues.Add(term[0]);
queue.AddDelete(term);
if (Random().Next(20) == 0 || j == ids.Length - 1)
{
queue.UpdateSlice(slice1);
Assert.IsTrue(slice1.IsTailItem(term));
slice1.Apply(bd1, j);
AssertAllBetween(last1, j, bd1, ids);
last1 = j + 1;
}
if (Random().Next(10) == 5 || j == ids.Length - 1)
{
queue.UpdateSlice(slice2);
Assert.IsTrue(slice2.IsTailItem(term));
slice2.Apply(bd2, j);
AssertAllBetween(last2, j, bd2, ids);
last2 = j + 1;
}
Assert.AreEqual(j + 1, queue.NumGlobalTermDeletes());
}
Assert.AreEqual(uniqueValues, bd1.Terms_Nunit().Keys);
Assert.AreEqual(uniqueValues, bd2.Terms_Nunit().Keys);
HashSet<Term> frozenSet = new HashSet<Term>();
foreach (Term t in queue.FreezeGlobalBuffer(null).TermsIterable())
{
BytesRef bytesRef = new BytesRef();
bytesRef.CopyBytes(t.Bytes());
frozenSet.Add(new Term(t.Field(), bytesRef));
}
Assert.AreEqual(uniqueValues, frozenSet);
Assert.AreEqual(0, queue.NumGlobalTermDeletes(), "num deletes must be 0 after freeze");
}
public virtual void TestStressDeleteQueue()
{
DocumentsWriterDeleteQueue queue = new DocumentsWriterDeleteQueue();
HashSet <Term> uniqueValues = new HashSet <Term>();
int size = 10000 + Random().Next(500) * RANDOM_MULTIPLIER;
int?[] ids = new int?[size];
for (int i = 0; i < ids.Length; i++)
{
ids[i] = Random().Next();
uniqueValues.Add(new Term("id", ids[i].ToString()));
}
CountDownLatch latch = new CountDownLatch(1);
AtomicInteger index = new AtomicInteger(0);
int numThreads = 2 + Random().Next(5);
UpdateThread[] threads = new UpdateThread[numThreads];
for (int i = 0; i < threads.Length; i++)
{
threads[i] = new UpdateThread(queue, index, ids, latch);
threads[i].Start();
}
latch.countDown();
for (int i = 0; i < threads.Length; i++)
{
threads[i].Join();
}
foreach (UpdateThread updateThread in threads)
{
DeleteSlice slice = updateThread.Slice;
queue.UpdateSlice(slice);
BufferedUpdates deletes = updateThread.Deletes;
slice.Apply(deletes, BufferedUpdates.MAX_INT);
Assert.AreEqual(uniqueValues, deletes.Terms_Nunit().Keys);
}
queue.TryApplyGlobalSlice();
HashSet <Term> frozenSet = new HashSet <Term>();
foreach (Term t in queue.FreezeGlobalBuffer(null).TermsIterable())
{
BytesRef bytesRef = new BytesRef();
bytesRef.CopyBytes(t.Bytes());
frozenSet.Add(new Term(t.Field(), bytesRef));
}
Assert.AreEqual(0, queue.NumGlobalTermDeletes(), "num deletes must be 0 after freeze");
Assert.AreEqual(uniqueValues.Count, frozenSet.Count);
Assert.AreEqual(uniqueValues, frozenSet);
}
public virtual void TestStressDeleteQueue()
{
DocumentsWriterDeleteQueue queue = new DocumentsWriterDeleteQueue();
ISet <Term> uniqueValues = new JCG.HashSet <Term>();
int size = 10000 + Random.Next(500) * RandomMultiplier;
int?[] ids = new int?[size];
for (int i = 0; i < ids.Length; i++)
{
ids[i] = Random.Next();
uniqueValues.Add(new Term("id", ids[i].ToString()));
}
CountdownEvent latch = new CountdownEvent(1);
AtomicInt32 index = new AtomicInt32(0);
int numThreads = 2 + Random.Next(5);
UpdateThread[] threads = new UpdateThread[numThreads];
for (int i = 0; i < threads.Length; i++)
{
threads[i] = new UpdateThread(queue, index, ids, latch);
threads[i].Start();
}
latch.Signal();
for (int i = 0; i < threads.Length; i++)
{
threads[i].Join();
}
foreach (UpdateThread updateThread in threads)
{
DeleteSlice slice = updateThread.slice;
queue.UpdateSlice(slice);
BufferedUpdates deletes = updateThread.deletes;
slice.Apply(deletes, BufferedUpdates.MAX_INT32);
assertEquals(uniqueValues, deletes.terms.Keys);
}
queue.TryApplyGlobalSlice();
ISet <Term> frozenSet = new JCG.HashSet <Term>();
foreach (Term t in queue.FreezeGlobalBuffer(null).GetTermsEnumerable())
{
BytesRef bytesRef = new BytesRef();
bytesRef.CopyBytes(t.Bytes);
frozenSet.Add(new Term(t.Field, bytesRef));
}
Assert.AreEqual(0, queue.NumGlobalTermDeletes, "num deletes must be 0 after freeze");
Assert.AreEqual(uniqueValues.Count, frozenSet.Count);
assertEquals(uniqueValues, frozenSet);
}
internal FlushedSegment(SegmentCommitInfo segmentInfo, FieldInfos fieldInfos, BufferedUpdates segmentUpdates, MutableBits liveDocs, int delCount)
{
this.SegmentInfo = segmentInfo;
this.FieldInfos = fieldInfos;
this.SegmentUpdates = segmentUpdates != null && segmentUpdates.Any() ? new FrozenBufferedUpdates(segmentUpdates, true) : null;
this.LiveDocs = liveDocs;
this.DelCount = delCount;
}
public DocumentsWriterPerThread(string segmentName, Directory directory, LiveIndexWriterConfig indexWriterConfig, InfoStream infoStream, DocumentsWriterDeleteQueue deleteQueue, FieldInfos.Builder fieldInfos)
{
this.DirectoryOrig = directory;
this.Directory = new TrackingDirectoryWrapper(directory);
this.FieldInfos = fieldInfos;
this.IndexWriterConfig = indexWriterConfig;
this.InfoStream = infoStream;
this.Codec = indexWriterConfig.Codec;
this.docState = new DocState(this, infoStream);
this.docState.Similarity = indexWriterConfig.Similarity;
bytesUsed = Counter.NewCounter();
ByteBlockAllocator = new DirectTrackingAllocator(bytesUsed);
PendingUpdates = new BufferedUpdates();
intBlockAllocator = new IntBlockAllocator(bytesUsed);
this.DeleteQueue = deleteQueue;
Debug.Assert(numDocsInRAM == 0, "num docs " + numDocsInRAM);
PendingUpdates.Clear();
DeleteSlice = deleteQueue.NewSlice();
SegmentInfo_Renamed = new SegmentInfo(DirectoryOrig, Constants.LUCENE_MAIN_VERSION, segmentName, -1, false, Codec, null);
Debug.Assert(numDocsInRAM == 0);
if (INFO_VERBOSE && infoStream.IsEnabled("DWPT"))
{
infoStream.Message("DWPT", Thread.CurrentThread.Name + " init seg=" + segmentName + " delQueue=" + deleteQueue);
}
// this should be the last call in the ctor
// it really sucks that we need to pull this within the ctor and pass this ref to the chain!
Consumer = indexWriterConfig.IndexingChain.GetChain(this);
}
/// <summary>
/// Create a shallow copy of <seealso cref="SegmentWriteState"/> with a new segment suffix. </summary>
public SegmentWriteState(SegmentWriteState state, string segmentSuffix)
{
InfoStream = state.InfoStream;
Directory = state.Directory;
SegmentInfo = state.SegmentInfo;
FieldInfos = state.FieldInfos;
TermIndexInterval = state.TermIndexInterval;
Context = state.Context;
this.SegmentSuffix = segmentSuffix;
SegUpdates = state.SegUpdates;
DelCountOnFlush = state.DelCountOnFlush;
}
internal override void Apply(BufferedUpdates bufferedUpdates, int docIDUpto)
{
bufferedUpdates.AddNumericUpdate((NumericDocValuesUpdate)item, docIDUpto);
}
//internal static readonly AtomicReferenceFieldUpdater<Node, Node> NextUpdater = AtomicReferenceFieldUpdater.newUpdater(typeof(Node), typeof(Node), "next");
internal virtual void Apply(BufferedUpdates bufferedDeletes, int docIDUpto)
{
throw new InvalidOperationException("sentinel item must never be applied");
}
internal FlushedSegment(SegmentCommitInfo segmentInfo, FieldInfos fieldInfos, BufferedUpdates segmentUpdates, IMutableBits liveDocs, int delCount)
{
this.segmentInfo = segmentInfo;
this.fieldInfos = fieldInfos;
this.segmentUpdates = segmentUpdates != null && segmentUpdates.Any() ? new FrozenBufferedUpdates(segmentUpdates, true) : null;
this.liveDocs = liveDocs;
this.delCount = delCount;
}
/// <summary>
/// Sole constructor. </summary>
public SegmentWriteState(InfoStream infoStream, Directory directory, SegmentInfo segmentInfo, FieldInfos fieldInfos, int termIndexInterval, BufferedUpdates segUpdates, IOContext context)
: this(infoStream, directory, segmentInfo, fieldInfos, termIndexInterval, segUpdates, context, "")
{
}
internal override void Apply(BufferedUpdates bufferedUpdates, int docIDUpto)
{
bufferedUpdates.AddNumericUpdate((NumericDocValuesUpdate)Item, docIDUpto);
}
internal override void Apply(BufferedUpdates bufferedUpdates, int docIDUpto)
{
foreach (Term term in (Term[])Item)
{
bufferedUpdates.AddTerm(term, docIDUpto);
}
}
internal override void Apply(BufferedUpdates bufferedUpdates, int docIDUpto)
{
foreach (Query query in (Query[])Item)
{
bufferedUpdates.AddQuery(query, docIDUpto);
}
}
internal override void Apply(BufferedUpdates bufferedDeletes, int docIDUpto)
{
bufferedDeletes.AddTerm((Term)Item, docIDUpto);
}
//internal static readonly AtomicReferenceFieldUpdater<Node, Node> NextUpdater = AtomicReferenceFieldUpdater.newUpdater(typeof(Node), typeof(Node), "next");
internal virtual void Apply(BufferedUpdates bufferedDeletes, int docIDUpto)
{
throw new InvalidOperationException("sentinel item must never be applied");
}
public virtual void Apply(BufferedUpdates del, int docIDUpto)
{
if (SliceHead == SliceTail)
{
// 0 length slice
return;
}
/*
* When we apply a slice we take the head and get its next as our first
* item to apply and continue until we applied the tail. If the head and
* tail in this slice are not equal then there will be at least one more
* non-null node in the slice!
*/
Node current = SliceHead;
do
{
current = current.Next;
Debug.Assert(current != null, "slice property violated between the head on the tail must not be a null node");
current.Apply(del, docIDUpto);
// System.out.println(Thread.currentThread().getName() + ": pull " + current + " docIDUpto=" + docIDUpto);
} while (current != SliceTail);
Reset();
}
internal readonly bool IsSegmentPrivate; // set to true iff this frozen packet represents
// a segment private deletes. in that case is should
// only have Queries
public FrozenBufferedUpdates(BufferedUpdates deletes, bool isSegmentPrivate)
{
this.IsSegmentPrivate = isSegmentPrivate;
Debug.Assert(!isSegmentPrivate || deletes.Terms.Count == 0, "segment private package should only have del queries");
Term[] termsArray = deletes.Terms.Keys.ToArray(/*new Term[deletes.Terms.Count]*/);
termCount = termsArray.Length;
ArrayUtil.TimSort(termsArray);
PrefixCodedTerms.Builder builder = new PrefixCodedTerms.Builder();
foreach (Term term in termsArray)
{
builder.Add(term);
}
Terms = builder.Finish();
Queries = new Query[deletes.Queries.Count];
QueryLimits = new int?[deletes.Queries.Count];
int upto = 0;
foreach (KeyValuePair <Query, int?> ent in deletes.Queries)
{
Queries[upto] = ent.Key;
QueryLimits[upto] = ent.Value;
upto++;
}
// TODO if a Term affects multiple fields, we could keep the updates key'd by Term
// so that it maps to all fields it affects, sorted by their docUpto, and traverse
// that Term only once, applying the update to all fields that still need to be
// updated.
IList <NumericDocValuesUpdate> allNumericUpdates = new List <NumericDocValuesUpdate>();
int numericUpdatesSize = 0;
foreach (/*Linked*/ HashMap <Term, NumericDocValuesUpdate> numericUpdates in deletes.NumericUpdates.Values)
{
foreach (NumericDocValuesUpdate update in numericUpdates.Values)
{
allNumericUpdates.Add(update);
numericUpdatesSize += update.SizeInBytes();
}
}
NumericDVUpdates = allNumericUpdates.ToArray();
// TODO if a Term affects multiple fields, we could keep the updates key'd by Term
// so that it maps to all fields it affects, sorted by their docUpto, and traverse
// that Term only once, applying the update to all fields that still need to be
// updated.
IList <BinaryDocValuesUpdate> allBinaryUpdates = new List <BinaryDocValuesUpdate>();
int binaryUpdatesSize = 0;
foreach (/*Linked*/ HashMap <Term, BinaryDocValuesUpdate> binaryUpdates in deletes.BinaryUpdates.Values)
{
foreach (BinaryDocValuesUpdate update in binaryUpdates.Values)
{
allBinaryUpdates.Add(update);
binaryUpdatesSize += update.SizeInBytes();
}
}
BinaryDVUpdates = allBinaryUpdates.ToArray();
BytesUsed = (int)Terms.SizeInBytes + Queries.Length * BYTES_PER_DEL_QUERY + numericUpdatesSize + NumericDVUpdates.Length * RamUsageEstimator.NUM_BYTES_OBJECT_REF + binaryUpdatesSize + BinaryDVUpdates.Length * RamUsageEstimator.NUM_BYTES_OBJECT_REF;
NumTermDeletes = deletes.NumTermDeletes.Get();
}
internal override void Apply(BufferedUpdates bufferedUpdates, int docIDUpto)
{
bufferedUpdates.AddBinaryUpdate((BinaryDocValuesUpdate)Item, docIDUpto);
}
//internal static readonly AtomicReferenceFieldUpdater<Node, Node> NextUpdater = AtomicReferenceFieldUpdater.newUpdater(typeof(Node), typeof(Node), "next");
internal virtual void Apply(BufferedUpdates bufferedDeletes, int docIDUpto)
{
throw IllegalStateException.Create("sentinel item must never be applied");
}
public DocumentsWriterDeleteQueue(BufferedUpdates globalBufferedUpdates, long generation)
{
this.GlobalBufferedUpdates = globalBufferedUpdates;
this.Generation = generation;
/*
* we use a sentinel instance as our initial tail. No slice will ever try to
* apply this tail since the head is always omitted.
*/
Tail = new Node(null); // sentinel
GlobalSlice = new DeleteSlice(Tail);
}
/// <summary>
/// Constructor which takes segment suffix.
/// </summary>
/// <seealso cref="SegmentWriteState(InfoStream, Directory, SegmentInfo, FieldInfos, int,
/// BufferedUpdates, IOContext)"/>
public SegmentWriteState(InfoStream infoStream, Directory directory, SegmentInfo segmentInfo, FieldInfos fieldInfos, int termIndexInterval, BufferedUpdates segUpdates, IOContext context, string segmentSuffix)
{
this.InfoStream = infoStream;
this.SegUpdates = segUpdates;
this.Directory = directory;
this.SegmentInfo = segmentInfo;
this.FieldInfos = fieldInfos;
this.TermIndexInterval = termIndexInterval;
this.SegmentSuffix = segmentSuffix;
this.Context = context;
}
protected internal UpdateThread(DocumentsWriterDeleteQueue queue, AtomicInteger index, int?[] ids, CountDownLatch latch)
{
this.Queue = queue;
this.Index = index;
this.Ids = ids;
this.Slice = queue.NewSlice();
Deletes = new BufferedUpdates();
this.Latch = latch;
}
internal readonly bool isSegmentPrivate; // set to true iff this frozen packet represents
// a segment private deletes. in that case is should
// only have Queries
public FrozenBufferedUpdates(BufferedUpdates deletes, bool isSegmentPrivate)
{
this.isSegmentPrivate = isSegmentPrivate;
Debug.Assert(!isSegmentPrivate || deletes.terms.Count == 0, "segment private package should only have del queries");
Term[] termsArray = deletes.terms.Keys.ToArray(/*new Term[deletes.Terms.Count]*/);
termCount = termsArray.Length;
ArrayUtil.TimSort(termsArray);
PrefixCodedTerms.Builder builder = new PrefixCodedTerms.Builder();
foreach (Term term in termsArray)
{
builder.Add(term);
}
terms = builder.Finish();
queries = new Query[deletes.queries.Count];
queryLimits = new int[deletes.queries.Count];
int upto = 0;
foreach (KeyValuePair <Query, int?> ent in deletes.queries)
{
queries[upto] = ent.Key;
if (ent.Value.HasValue)
{
queryLimits[upto] = ent.Value.Value;
}
else
{
// LUCENENET NOTE: According to this: http://stackoverflow.com/a/13914344
// we are supposed to throw an exception in this case, rather than
// silently fail.
throw new NullReferenceException();
}
upto++;
}
// TODO if a Term affects multiple fields, we could keep the updates key'd by Term
// so that it maps to all fields it affects, sorted by their docUpto, and traverse
// that Term only once, applying the update to all fields that still need to be
// updated.
IList <NumericDocValuesUpdate> allNumericUpdates = new List <NumericDocValuesUpdate>();
int numericUpdatesSize = 0;
foreach (var numericUpdates in deletes.numericUpdates.Values)
{
foreach (NumericDocValuesUpdate update in numericUpdates.Values)
{
allNumericUpdates.Add(update);
numericUpdatesSize += update.GetSizeInBytes();
}
}
numericDVUpdates = allNumericUpdates.ToArray();
// TODO if a Term affects multiple fields, we could keep the updates key'd by Term
// so that it maps to all fields it affects, sorted by their docUpto, and traverse
// that Term only once, applying the update to all fields that still need to be
// updated.
IList <BinaryDocValuesUpdate> allBinaryUpdates = new List <BinaryDocValuesUpdate>();
int binaryUpdatesSize = 0;
foreach (var binaryUpdates in deletes.binaryUpdates.Values)
{
foreach (BinaryDocValuesUpdate update in binaryUpdates.Values)
{
allBinaryUpdates.Add(update);
binaryUpdatesSize += update.GetSizeInBytes();
}
}
binaryDVUpdates = allBinaryUpdates.ToArray();
bytesUsed = (int)terms.GetSizeInBytes() + queries.Length * BYTES_PER_DEL_QUERY + numericUpdatesSize + numericDVUpdates.Length * RamUsageEstimator.NUM_BYTES_OBJECT_REF + binaryUpdatesSize + binaryDVUpdates.Length * RamUsageEstimator.NUM_BYTES_OBJECT_REF;
numTermDeletes = deletes.numTermDeletes.Get();
}
/// <summary>
/// Sole constructor. </summary>
public SegmentWriteState(InfoStream infoStream, Directory directory, SegmentInfo segmentInfo, FieldInfos fieldInfos, int termIndexInterval, BufferedUpdates segUpdates, IOContext context)
: this(infoStream, directory, segmentInfo, fieldInfos, termIndexInterval, segUpdates, context, "")
{
}
internal override void Apply(BufferedUpdates bufferedDeletes, int docIDUpto)
{
bufferedDeletes.AddTerm((Term)item, docIDUpto);
}
private void AssertAllBetween(int start, int end, BufferedUpdates deletes, int?[] ids)
{
for (int i = start; i <= end; i++)
{
Assert.AreEqual(Convert.ToInt32(end), deletes.Terms_Nunit()[new Term("id", ids[i].ToString())]);
}
}
internal override void Apply(BufferedUpdates bufferedUpdates, int docIDUpto)
{
bufferedUpdates.AddBinaryUpdate((BinaryDocValuesUpdate)item, docIDUpto);
}
/// <summary>
/// Constructor which takes segment suffix.
/// </summary>
/// <seealso cref= #SegmentWriteState(InfoStream, Directory, SegmentInfo, FieldInfos, int,
/// BufferedUpdates, IOContext) </seealso>
public SegmentWriteState(InfoStream infoStream, Directory directory, SegmentInfo segmentInfo, FieldInfos fieldInfos, int termIndexInterval, BufferedUpdates segUpdates, IOContext context, string segmentSuffix)
{
this.InfoStream = infoStream;
this.SegUpdates = segUpdates;
this.Directory = directory;
this.SegmentInfo = segmentInfo;
this.FieldInfos = fieldInfos;
this.TermIndexInterval = termIndexInterval;
this.SegmentSuffix = segmentSuffix;
this.Context = context;
}