/// <summary>
/// Appends a new packet of buffered deletes to the stream,
/// setting its generation:
/// </summary>
public virtual long Push(FrozenBufferedUpdates packet)
{
lock (this)
{
/*
* The insert operation must be atomic. If we let threads increment the gen
* and push the packet afterwards we risk that packets are out of order.
* With DWPT this is possible if two or more flushes are racing for pushing
* updates. If the pushed packets get our of order would loose documents
* since deletes are applied to the wrong segments.
*/
packet.DelGen = nextGen++;
Debug.Assert(packet.Any());
Debug.Assert(CheckDeleteStats());
Debug.Assert(packet.DelGen < nextGen);
Debug.Assert(updates.Count == 0 || updates[updates.Count - 1].DelGen < packet.DelGen, "Delete packets must be in order");
updates.Add(packet);
numTerms.AddAndGet(packet.numTermDeletes);
bytesUsed.AddAndGet(packet.bytesUsed);
if (infoStream.IsEnabled("BD"))
{
infoStream.Message("BD", "push deletes " + packet + " delGen=" + packet.DelGen + " packetCount=" + updates.Count + " totBytesUsed=" + bytesUsed.Get());
}
Debug.Assert(CheckDeleteStats());
return(packet.DelGen);
}
}
internal bool UpdateDocuments(IEnumerable <IEnumerable <IIndexableField> > docs, Analyzer analyzer, Term delTerm)
{
bool hasEvents = PreUpdate();
ThreadState perThread = flushControl.ObtainAndLock();
DocumentsWriterPerThread flushingDWPT;
try
{
if (!perThread.IsActive)
{
EnsureOpen();
if (Debugging.AssertsEnabled)
{
Debugging.Assert(false, () => "perThread is not active but we are still open");
}
}
EnsureInitialized(perThread);
if (Debugging.AssertsEnabled)
{
Debugging.Assert(perThread.IsInitialized);
}
DocumentsWriterPerThread dwpt = perThread.dwpt;
int dwptNumDocs = dwpt.NumDocsInRAM;
try
{
int docCount = dwpt.UpdateDocuments(docs, analyzer, delTerm);
numDocsInRAM.AddAndGet(docCount);
}
finally
{
if (dwpt.CheckAndResetHasAborted())
{
if (dwpt.PendingFilesToDelete.Count > 0)
{
PutEvent(new DeleteNewFilesEvent(dwpt.PendingFilesToDelete));
}
SubtractFlushedNumDocs(dwptNumDocs);
flushControl.DoOnAbort(perThread);
}
}
bool isUpdate = delTerm != null;
flushingDWPT = flushControl.DoAfterDocument(perThread, isUpdate);
}
finally
{
perThread.Unlock();
}
return(PostUpdate(flushingDWPT, hasEvents));
}
public virtual void TestBooleanScorerMax()
{
Directory dir = NewDirectory();
RandomIndexWriter riw = new RandomIndexWriter(Random, dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random)));
int docCount = AtLeast(10000);
for (int i = 0; i < docCount; i++)
{
Document doc = new Document();
doc.Add(NewField("field", "a", TextField.TYPE_NOT_STORED));
riw.AddDocument(doc);
}
riw.ForceMerge(1);
IndexReader r = riw.GetReader();
riw.Dispose();
IndexSearcher s = NewSearcher(r);
BooleanQuery bq = new BooleanQuery();
bq.Add(new TermQuery(new Term("field", "a")), Occur.SHOULD);
bq.Add(new TermQuery(new Term("field", "a")), Occur.SHOULD);
Weight w = s.CreateNormalizedWeight(bq);
Assert.AreEqual(1, s.IndexReader.Leaves.Count);
BulkScorer scorer = w.GetBulkScorer(s.IndexReader.Leaves[0], false, null);
FixedBitSet hits = new FixedBitSet(docCount);
AtomicInt32 end = new AtomicInt32();
ICollector c = new CollectorAnonymousClass(this, scorer, hits, end);
while (end < docCount)
{
int inc = TestUtil.NextInt32(Random, 1, 1000);
end.AddAndGet(inc);
scorer.Score(c, end);
}
Assert.AreEqual(docCount, hits.Cardinality);
r.Dispose();
dir.Dispose();
}
public override void Run()
{
if (VERBOSE)
{
Console.WriteLine(Thread.CurrentThread.Name + ": launch search thread");
}
while (Environment.TickCount < stopTimeMS)
{
try
{
IndexSearcher s = outerInstance.GetCurrentSearcher();
try
{
// Verify 1) IW is correctly setting
// diagnostics, and 2) segment warming for
// merged segments is actually happening:
foreach (AtomicReaderContext sub in s.IndexReader.Leaves)
{
SegmentReader segReader = (SegmentReader)sub.Reader;
IDictionary <string, string> diagnostics = segReader.SegmentInfo.Info.Diagnostics;
assertNotNull(diagnostics);
string source;
diagnostics.TryGetValue("source", out source);
assertNotNull(source);
if (source.Equals("merge", StringComparison.Ordinal))
{
assertTrue("sub reader " + sub + " wasn't warmed: warmed=" + outerInstance.warmed + " diagnostics=" + diagnostics + " si=" + segReader.SegmentInfo,
!outerInstance.m_assertMergedSegmentsWarmed || outerInstance.warmed.ContainsKey(segReader.core));
}
}
if (s.IndexReader.NumDocs > 0)
{
outerInstance.SmokeTestSearcher(s);
Fields fields = MultiFields.GetFields(s.IndexReader);
if (fields == null)
{
continue;
}
Terms terms = fields.GetTerms("body");
if (terms == null)
{
continue;
}
TermsEnum termsEnum = terms.GetIterator(null);
int seenTermCount = 0;
int shift;
int trigger;
if (totTermCount.Get() < 30)
{
shift = 0;
trigger = 1;
}
else
{
trigger = totTermCount.Get() / 30;
shift = Random.Next(trigger);
}
while (Environment.TickCount < stopTimeMS)
{
BytesRef term = termsEnum.Next();
if (term == null)
{
totTermCount.Set(seenTermCount);
break;
}
seenTermCount++;
// search 30 terms
if ((seenTermCount + shift) % trigger == 0)
{
//if (VERBOSE) {
//System.out.println(Thread.currentThread().getName() + " now search body:" + term.Utf8ToString());
//}
totHits.AddAndGet(outerInstance.RunQuery(s, new TermQuery(new Term("body", term))));
}
}
//if (VERBOSE) {
//System.out.println(Thread.currentThread().getName() + ": search done");
//}
}
}
finally
{
outerInstance.ReleaseSearcher(s);
}
}
catch (Exception t)
{
Console.WriteLine(Thread.CurrentThread.Name + ": hit exc");
outerInstance.m_failed.Set(true);
Console.WriteLine(t.ToString());
throw new Exception(t.ToString(), t);
}
}
}
public override void Run()
{
if (Verbose)
{
Console.WriteLine(Thread.CurrentThread.Name + ": launch search thread");
}
while (J2N.Time.NanoTime() / J2N.Time.MillisecondsPerNanosecond < stopTimeMS) // LUCENENET: Use NanoTime() rather than CurrentTimeMilliseconds() for more accurate/reliable results
{
try
{
IndexSearcher s = outerInstance.GetCurrentSearcher();
try
{
// Verify 1) IW is correctly setting
// diagnostics, and 2) segment warming for
// merged segments is actually happening:
foreach (AtomicReaderContext sub in s.IndexReader.Leaves)
{
SegmentReader segReader = (SegmentReader)sub.Reader;
IDictionary <string, string> diagnostics = segReader.SegmentInfo.Info.Diagnostics;
assertNotNull(diagnostics);
diagnostics.TryGetValue("source", out string source);
assertNotNull(source);
if (source.Equals("merge", StringComparison.Ordinal))
{
assertTrue("sub reader " + sub + " wasn't warmed: warmed=" + outerInstance.warmed + " diagnostics=" + diagnostics + " si=" + segReader.SegmentInfo,
// LUCENENET: ConditionalWeakTable doesn't have ContainsKey, so we normalize to TryGetValue
!outerInstance.m_assertMergedSegmentsWarmed || outerInstance.warmed.TryGetValue(segReader.core, out BooleanRef _));
}
}
if (s.IndexReader.NumDocs > 0)
{
outerInstance.SmokeTestSearcher(s);
Fields fields = MultiFields.GetFields(s.IndexReader);
if (fields == null)
{
continue;
}
Terms terms = fields.GetTerms("body");
if (terms == null)
{
continue;
}
TermsEnum termsEnum = terms.GetEnumerator();
int seenTermCount = 0;
int shift;
int trigger;
if (totTermCount < 30)
{
shift = 0;
trigger = 1;
}
else
{
trigger = totTermCount / 30;
shift = Random.Next(trigger);
}
while (J2N.Time.NanoTime() / J2N.Time.MillisecondsPerNanosecond < stopTimeMS) // LUCENENET: Use NanoTime() rather than CurrentTimeMilliseconds() for more accurate/reliable results
{
if (!termsEnum.MoveNext())
{
totTermCount.Value = seenTermCount;
break;
}
seenTermCount++;
// search 30 terms
if ((seenTermCount + shift) % trigger == 0)
{
//if (VERBOSE) {
//System.out.println(Thread.currentThread().getName() + " now search body:" + term.Utf8ToString());
//}
totHits.AddAndGet(outerInstance.RunQuery(s, new TermQuery(new Term("body", termsEnum.Term))));
}
}
//if (VERBOSE) {
//System.out.println(Thread.currentThread().getName() + ": search done");
//}
}
}
finally
{
outerInstance.ReleaseSearcher(s);
}
}
catch (Exception t) when(t.IsThrowable())
{
Console.WriteLine(Thread.CurrentThread.Name + ": hit exc");
outerInstance.m_failed.Value = (true);
Console.WriteLine(t.ToString());
throw RuntimeException.Create(t);
}
}
}
