internal void put(File file)
{
int curCacheCount = cacheCount.Get();
while (curCacheCount + 1 > countLimit)
{
long freedSize = removeNext();
cacheSize.AddAndGet(-freedSize);
curCacheCount = cacheCount.AddAndGet(-1);
}
cacheCount.AddAndGet(1);
long valueSize = calculateSize(file);
long curCacheSize = cacheSize.Get();
while (curCacheSize + valueSize > sizeLimit)
{
long freedSize = removeNext();
curCacheSize = cacheSize.AddAndGet(-freedSize);
}
cacheSize.AddAndGet(valueSize);
long currentTime = ConvertUtils.GetCurrentTimeMillis();
file.SetLastModified(currentTime);
lastUsageDates.TryAdd(file, currentTime);
}
// Appends a new packet of buffered deletes to the stream,
// setting its generation:
public virtual long Push(FrozenBufferedUpdates packet)
{
lock (this)
{
/*
* The insert operation must be atomic. If we let threads increment the gen
* and push the packet afterwards we risk that packets are out of order.
* With DWPT this is possible if two or more flushes are racing for pushing
* updates. If the pushed packets get our of order would loose documents
* since deletes are applied to the wrong segments.
*/
packet.DelGen = NextGen_Renamed++;
Debug.Assert(packet.Any());
Debug.Assert(CheckDeleteStats());
Debug.Assert(packet.DelGen < NextGen_Renamed);
Debug.Assert(Updates.Count == 0 || Updates[Updates.Count - 1].DelGen < packet.DelGen, "Delete packets must be in order");
Updates.Add(packet);
numTerms.AddAndGet(packet.NumTermDeletes);
bytesUsed.AddAndGet(packet.BytesUsed);
if (InfoStream.IsEnabled("BD"))
{
InfoStream.Message("BD", "push deletes " + packet + " delGen=" + packet.DelGen + " packetCount=" + Updates.Count + " totBytesUsed=" + bytesUsed.Get());
}
Debug.Assert(CheckDeleteStats());
return(packet.DelGen);
}
}
public void TestThreadSafety()
{
var time = new MockedTime();
var p = new HystrixRollingPercentile(time, 100, 25, 1000, true);
int num_threads = 1000; // .NET Core StackOverflow
int num_iterations = 1000000;
var latch = new CountdownEvent(num_threads);
var aggregateMetrics = new AtomicInteger(); // same as a blackhole
var r = new Random();
var cts = new CancellationTokenSource();
var metricsPoller = Task.Run(() =>
{
while (!cts.Token.IsCancellationRequested)
{
aggregateMetrics.AddAndGet(p.Mean + p.GetPercentile(10) + p.GetPercentile(50) + p.GetPercentile(90));
}
});
for (int i = 0; i < num_threads; i++)
{
int threadId = i;
Task.Run(() =>
{
for (int j = 1; j < (num_iterations / num_threads) + 1; j++)
{
int nextInt = r.Next(100);
p.AddValue(nextInt);
if (threadId == 0)
{
time.Increment(1);
}
}
latch.SignalEx();
});
}
try
{
latch.Wait(TimeSpan.FromSeconds(100));
cts.Cancel();
}
catch (Exception)
{
Assert.True(false, "Timeout on all threads writing percentiles");
}
aggregateMetrics.AddAndGet(p.Mean + p.GetPercentile(10) + p.GetPercentile(50) + p.GetPercentile(90));
output.WriteLine(p.Mean + " : " + p.GetPercentile(50) + " : " + p.GetPercentile(75) + " : " + p.GetPercentile(90) + " : " + p.GetPercentile(95) + " : " + p.GetPercentile(99));
}
public AccessSafely AfterCompleting <TState, TSource>(int times)
{
_access = AccessSafely.AfterCompleting(times);
_access
.WritingWith <int>("confirmDispatchedResultedIn", increment => _confirmDispatchedResultedIn.AddAndGet(increment))
.ReadingWith("confirmDispatchedResultedIn", () => _confirmDispatchedResultedIn.Get())
.WritingWith <StoreData <TSource> >("writeStoreData", data =>
{
_writeObjectResultedIn.AddAndGet(data.ResultedIn);
_objectWriteResult.Set(data.Result);
_objectWriteAccumulatedResults.Enqueue(data.Result);
_objectState.Set(data.State);
data.Sources.ForEach(source => _sources.Enqueue(source));
_metadataHolder.Set(data.Metadata);
if (data.ErrorCauses != null)
{
_errorCauses.Enqueue(data.ErrorCauses);
}
})
.WritingWith <StoreData <TSource> >("readStoreData", data =>
{
_readObjectResultedIn.AddAndGet(data.ResultedIn);
_objectReadResult.Set(data.Result);
_objectWriteAccumulatedResults.Enqueue(data.Result);
_objectState.Set(data.State);
data.Sources.ForEach(source => _sources.Enqueue(source));
_metadataHolder.Set(data.Metadata);
if (data.ErrorCauses != null)
{
_errorCauses.Enqueue(data.ErrorCauses);
}
})
.ReadingWith("readObjectResultedIn", () => _readObjectResultedIn.Get())
.ReadingWith("objectReadResult", () => _objectReadResult.Get())
.ReadingWith("objectWriteResult", () => _objectWriteResult.Get())
.ReadingWith("objectWriteAccumulatedResults", () =>
{
_objectWriteAccumulatedResults.TryDequeue(out var result);
return(result);
})
.ReadingWith("objectWriteAccumulatedResultsCount", () => _objectWriteAccumulatedResults.Count)
.ReadingWith("metadataHolder", () => _metadataHolder.Get())
.ReadingWith("objectState", () => (TState)_objectState.Get())
.ReadingWith("sources", () =>
{
_sources.TryDequeue(out var result);
return(result);
})
.ReadingWith("errorCauses", () =>
{
_errorCauses.TryDequeue(out var result);
return(result);
})
.ReadingWith("errorCausesCount", () => _errorCauses.Count)
.ReadingWith("writeObjectResultedIn", () => _writeObjectResultedIn.Get());
return(_access);
}
private void ProcessSpeculatorEvent(SpeculatorEvent @event)
{
lock (this)
{
switch (@event.GetType())
{
case Speculator.EventType.AttemptStatusUpdate:
{
StatusUpdate(@event.GetReportedStatus(), @event.GetTimestamp());
break;
}
case Speculator.EventType.TaskContainerNeedUpdate:
{
AtomicInteger need = ContainerNeed(@event.GetTaskID());
need.AddAndGet(@event.ContainersNeededChange());
break;
}
case Speculator.EventType.AttemptStart:
{
Log.Info("ATTEMPT_START " + @event.GetTaskID());
estimator.EnrollAttempt(@event.GetReportedStatus(), @event.GetTimestamp());
break;
}
case Speculator.EventType.JobCreate:
{
Log.Info("JOB_CREATE " + @event.GetJobID());
estimator.Contextualize(GetConfig(), context);
break;
}
}
}
}
/// <summary>Increment the value by a delta</summary>
/// <param name="delta">of the increment</param>
public virtual void Incr(int delta)
{
lock (this)
{
value.AddAndGet(delta);
SetChanged();
}
}
public AccessSafely AfterCompleting(int times)
{
_access = AccessSafely.AfterCompleting(times);
_access
.WritingWith <int>("sentCount", increment => _sentCount.AddAndGet(increment))
.ReadingWith("sentCount", () => _sentCount.Get());
return(_access);
}
public Results(int times)
{
var one = new AtomicInteger(0);
var two = new AtomicInteger(0);
var three = new AtomicInteger(0);
counters = AccessSafely.AfterCompleting(times);
counters.WritingWith <int>("one", x => one.AddAndGet(x));
counters.ReadingWith("one", one.Get);
counters.WritingWith <int>("two", x => two.AddAndGet(x));
counters.ReadingWith("two", two.Get);
counters.WritingWith <int>("three", x => three.AddAndGet(x));
counters.ReadingWith("three", three.Get);
}
public virtual void TestBooleanScorerMax()
{
Directory dir = NewDirectory();
RandomIndexWriter riw = new RandomIndexWriter(Random(), dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())));
int docCount = AtLeast(10000);
for (int i = 0; i < docCount; i++)
{
Document doc = new Document();
doc.Add(NewField("field", "a", TextField.TYPE_NOT_STORED));
riw.AddDocument(doc);
}
riw.ForceMerge(1);
IndexReader r = riw.Reader;
riw.Dispose();
IndexSearcher s = NewSearcher(r);
BooleanQuery bq = new BooleanQuery();
bq.Add(new TermQuery(new Term("field", "a")), BooleanClause.Occur.SHOULD);
bq.Add(new TermQuery(new Term("field", "a")), BooleanClause.Occur.SHOULD);
Weight w = s.CreateNormalizedWeight(bq);
Assert.AreEqual(1, s.IndexReader.Leaves.Count);
BulkScorer scorer = w.BulkScorer(s.IndexReader.Leaves[0], false, null);
FixedBitSet hits = new FixedBitSet(docCount);
AtomicInteger end = new AtomicInteger();
Collector c = new CollectorAnonymousInnerClassHelper(this, scorer, hits, end);
while (end.Get() < docCount)
{
int inc = TestUtil.NextInt(Random(), 1, 1000);
end.AddAndGet(inc);
scorer.Score(c, end.Get());
}
Assert.AreEqual(docCount, hits.Cardinality());
r.Dispose();
dir.Dispose();
}
internal bool UpdateDocuments(IEnumerable <IEnumerable <IndexableField> > docs, Analyzer analyzer, Term delTerm)
{
bool hasEvents = PreUpdate();
ThreadState perThread = FlushControl.ObtainAndLock();
DocumentsWriterPerThread flushingDWPT;
try
{
if (!perThread.Active)
{
EnsureOpen();
Debug.Assert(false, "perThread is not active but we are still open");
}
EnsureInitialized(perThread);
Debug.Assert(perThread.Initialized);
DocumentsWriterPerThread dwpt = perThread.Dwpt;
int dwptNumDocs = dwpt.NumDocsInRAM;
try
{
int docCount = dwpt.UpdateDocuments(docs, analyzer, delTerm);
NumDocsInRAM.AddAndGet(docCount);
}
finally
{
if (dwpt.CheckAndResetHasAborted())
{
if (dwpt.PendingFilesToDelete().Count > 0)
{
PutEvent(new DeleteNewFilesEvent(dwpt.PendingFilesToDelete()));
}
SubtractFlushedNumDocs(dwptNumDocs);
FlushControl.DoOnAbort(perThread);
}
}
bool isUpdate = delTerm != null;
flushingDWPT = FlushControl.DoAfterDocument(perThread, isUpdate);
}
finally
{
perThread.Unlock();
}
return(PostUpdate(flushingDWPT, hasEvents));
}
public override void Run()
{
if (VERBOSE)
{
Console.WriteLine(Thread.CurrentThread.Name + ": launch search thread");
}
while (DateTime.UtcNow < StopTime)
{
try
{
IndexSearcher s = OuterInstance.CurrentSearcher;
try
{
// Verify 1) IW is correctly setting
// diagnostics, and 2) segment warming for
// merged segments is actually happening:
foreach (AtomicReaderContext sub in s.IndexReader.Leaves)
{
SegmentReader segReader = (SegmentReader)sub.Reader;
IDictionary <string, string> diagnostics = segReader.SegmentInfo.Info.Diagnostics;
Assert.IsNotNull(diagnostics);
string source = diagnostics["source"];
Assert.IsNotNull(source);
if (source.Equals("merge"))
{
Assert.IsTrue(!OuterInstance.AssertMergedSegmentsWarmed || OuterInstance.Warmed.ContainsKey((SegmentCoreReaders)segReader.CoreCacheKey), "sub reader " + sub + " wasn't warmed: warmed=" + OuterInstance.Warmed + " diagnostics=" + diagnostics + " si=" + segReader.SegmentInfo);
}
}
if (s.IndexReader.NumDocs > 0)
{
OuterInstance.SmokeTestSearcher(s);
Fields fields = MultiFields.GetFields(s.IndexReader);
if (fields == null)
{
continue;
}
Terms terms = fields.Terms("body");
if (terms == null)
{
continue;
}
TermsEnum termsEnum = terms.Iterator(null);
int seenTermCount = 0;
int shift;
int trigger;
if (TotTermCount.Get() < 30)
{
shift = 0;
trigger = 1;
}
else
{
trigger = TotTermCount.Get() / 30;
shift = Random().Next(trigger);
}
while (DateTime.UtcNow < StopTime)
{
BytesRef term = termsEnum.Next();
if (term == null)
{
TotTermCount.Set(seenTermCount);
break;
}
seenTermCount++;
// search 30 terms
if ((seenTermCount + shift) % trigger == 0)
{
//if (VERBOSE) {
//System.out.println(Thread.currentThread().getName() + " now search body:" + term.Utf8ToString());
//}
TotHits.AddAndGet(OuterInstance.RunQuery(s, new TermQuery(new Term("body", term))));
}
}
//if (VERBOSE) {
//System.out.println(Thread.currentThread().getName() + ": search done");
//}
}
}
finally
{
OuterInstance.ReleaseSearcher(s);
}
}
catch (Exception t)
{
Console.WriteLine(Thread.CurrentThread.Name + ": hit exc");
OuterInstance.Failed.Set(true);
Console.WriteLine(t.StackTrace);
throw new Exception(t.Message, t);
}
}
}
/// <summary>Notify the monitor of workers starting.</summary>
/// <remarks>Notify the monitor of workers starting.</remarks>
/// <param name="count">the number of worker threads that are starting.</param>
public virtual void StartWorkers(int count)
{
workers.AddAndGet(count);
}
public void Increment(int millis)
{
time.AddAndGet(millis);
}
public static int addAndGet(this AtomicInteger atomicInteger, int delta)
{
return(atomicInteger.AddAndGet(delta));
}
