public static AtomicBoolean FromValue (bool value)
{
AtomicBoolean temp = new AtomicBoolean ();
temp.Value = value;
return temp;
}
public WidgetBuilderThread(WidgetBuilder widgetBuilder, string text, ParentWidget parent, AtomicBoolean failFlag)
{
this.widgetBuilder = widgetBuilder;
this.text = text;
this.parent = parent;
this.failFlag = failFlag;
}
public static void Main()
{
ComputerSpecifications.Dump();
var running = new AtomicBoolean(true);
using (var aeron = Aeron.Connect())
using (var publication = aeron.AddPublication(Channel, StreamID))
using (var subscription = aeron.AddSubscription(Channel, StreamID))
{
var subscriber = new Subscriber(running, subscription);
var subscriberThread = new Thread(subscriber.Run) {Name = "subscriber"};
var publisherThread = new Thread(new Publisher(running, publication).Run) {Name = "publisher"};
var rateReporterThread = new Thread(new RateReporter(running, subscriber).Run) {Name = "rate-reporter"};
rateReporterThread.Start();
subscriberThread.Start();
publisherThread.Start();
Console.WriteLine("Press any key to stop...");
Console.Read();
running.Set(false);
subscriberThread.Join();
publisherThread.Join();
rateReporterThread.Join();
}
}
public static void Main()
{
Console.WriteLine("Subscribing to " + Channel + " on stream Id " + StreamID);
var ctx = new Aeron.Context()
.AvailableImageHandler(SamplesUtil.PrintUnavailableImage)
.UnavailableImageHandler(SamplesUtil.PrintUnavailableImage);
var reporter = new RateReporter(1000, SamplesUtil.PrintRate);
var fragmentAssembler = new FragmentAssembler(SamplesUtil.RateReporterHandler(reporter));
var running = new AtomicBoolean(true);
var t = new Thread(subscription => SamplesUtil.SubscriberLoop(fragmentAssembler.OnFragment, FragmentCountLimit, running)((Subscription) subscription));
var report = new Thread(reporter.Run);
using (var aeron = Aeron.Connect(ctx))
using (var subscription = aeron.AddSubscription(Channel, StreamID))
{
t.Start(subscription);
report.Start();
Console.ReadLine();
Console.WriteLine("Shutting down...");
running.Set(false);
reporter.Halt();
t.Join();
report.Join();
}
}
/// <summary>
/// Creates a new <c>SingleThreadExecutor</c> using a custom thread priority.
/// </summary>
/// <param name="priority">
/// The priority to assign the thread.
/// </param>
public SingleThreadExecutor(ThreadPriority priority) {
_actions = new BlockingQueue<Action>();
_running = new AtomicBoolean(false);
_shuttingDown = new AtomicBoolean(false);
_priority = priority;
}
public void AtomicBoolean_Load_Should_Success()
{
var atomicBoolean = new AtomicBoolean(true);
Assert.Equal(true, atomicBoolean.Load(MemoryOrder.Relaxed));
Assert.Equal(true, atomicBoolean.Load(MemoryOrder.Acquire));
Assert.Equal(true, atomicBoolean.Load(MemoryOrder.AcqRel));
Assert.Equal(true, atomicBoolean.Load(MemoryOrder.SeqCst));
}
/// <summary>
/// Extracts a <see cref="Boolean"/> from an instance of <see cref="AtomicBoolean"/>.
/// </summary>
/// <param name="atomicBoolean">The <see cref="AtomicBoolean"/> to extract the value of.</param>
/// <exception cref="ArgumentNullException">
/// Thrown if <paramref name="atomicBoolean" /> is <see langword="null"/>.
/// </exception>
/// <returns>the value of the <see cref="AtomicBoolean"/>.</returns>
public static bool ToBoolean(AtomicBoolean atomicBoolean)
{
if (atomicBoolean == null)
{
throw new InvalidCastException();
}
return atomicBoolean.Value;
}
public IndexerThread(IndexWriter w, FacetsConfig config, TaxonomyWriter tw, ReferenceManager<SearcherAndTaxonomy> mgr, int ordLimit, AtomicBoolean stop)
{
this.w = w;
this.config = config;
this.tw = tw;
this.mgr = mgr;
this.ordLimit = ordLimit;
this.stop = stop;
}
public Result<bool> Execute(IAggregateRootId aggregateRootId, int aggregateRootRevision, Action action)
{
var result = new Result<bool>(false);
var acquired = new AtomicBoolean(false);
try
{
if (aggregateLock.TryGetValue(aggregateRootId, out acquired) == false)
{
acquired = acquired ?? new AtomicBoolean(false);
if (aggregateLock.TryAdd(aggregateRootId, acquired) == false)
aggregateLock.TryGetValue(aggregateRootId, out acquired);
}
if (acquired.CompareAndSet(false, true))
{
try
{
AtomicInteger revision = null;
if (aggregateRevisions.TryGetValue(aggregateRootId, out revision) == false)
{
revision = new AtomicInteger(aggregateRootRevision - 1);
if (aggregateRevisions.TryAdd(aggregateRootId, revision) == false)
return result;
}
var currentRevision = revision.Value;
if (revision.CompareAndSet(aggregateRootRevision - 1, aggregateRootRevision))
{
try
{
action();
return Result.Success;
}
catch (Exception)
{
revision.GetAndSet(currentRevision);
throw;
}
}
}
finally
{
acquired.GetAndSet(false);
}
}
return result;
}
catch (Exception ex)
{
return result.WithError(ex);
}
}
public void CompareExpectedValueAndSetNewValue()
{
AtomicBoolean ai = new AtomicBoolean( true );
Assert.AreEqual( true, ai.Value );
Assert.IsTrue( ai.CompareAndSet( true, false ) );
Assert.AreEqual( false, ai.Value );
Assert.IsTrue( ai.CompareAndSet( false, false ) );
Assert.AreEqual( false, ai.Value );
Assert.IsFalse( ai.CompareAndSet( true, false ) );
Assert.IsFalse( ( ai.Value ) );
Assert.IsTrue( ai.CompareAndSet( false, true ) );
Assert.AreEqual( true, ai.Value );
}
public void CompareExpectedValueAndSetNewValueInMultipleThreads()
{
AtomicBoolean ai = new AtomicBoolean( true );
var t = ThreadManager.StartAndAssertRegistered("T1",delegate
{
while (!ai.CompareAndSet(false, true))
Thread.Sleep(Delays.Short);
}
);
Assert.IsTrue( ai.CompareAndSet( true, false ), "Value" );
ThreadManager.JoinAndVerify();
Assert.IsFalse( t.IsAlive, "Thread is still alive." );
Assert.IsTrue( ai.Value );
}
public void TestBlockingReceiveWithTimeout()
{
QueueChannel channel = new QueueChannel();
AtomicBoolean receiveInterrupted = new AtomicBoolean(false);
CountDownLatch latch = new CountDownLatch(1);
Thread t = new Thread(new ThreadStart(delegate
{
IMessage message = channel.Receive(new TimeSpan(10000));
receiveInterrupted.Value = true;
Assert.IsTrue(message == null);
latch.CountDown();
}));
t.Start();
//Assert.IsFalse(receiveInterrupted.Value);
t.Interrupt();
latch.Await();
Assert.IsTrue(receiveInterrupted.Value);
}
public virtual void TestApplyDeletesOnFlush()
{
Directory dir = NewDirectory();
// Cannot use RandomIndexWriter because we don't want to
// ever call commit() for this test:
AtomicInteger docsInSegment = new AtomicInteger();
AtomicBoolean closing = new AtomicBoolean();
AtomicBoolean sawAfterFlush = new AtomicBoolean();
IndexWriter w = new IndexWriterAnonymousInnerClassHelper(this, dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetRAMBufferSizeMB(0.5).SetMaxBufferedDocs(-1).SetMergePolicy(NoMergePolicy.NO_COMPOUND_FILES).SetReaderPooling(false), docsInSegment, closing, sawAfterFlush);
int id = 0;
while (true)
{
StringBuilder sb = new StringBuilder();
for (int termIDX = 0; termIDX < 100; termIDX++)
{
sb.Append(' ').Append(TestUtil.RandomRealisticUnicodeString(Random()));
}
if (id == 500)
{
w.DeleteDocuments(new Term("id", "0"));
}
Document doc = new Document();
doc.Add(NewStringField("id", "" + id, Field.Store.NO));
doc.Add(NewTextField("body", sb.ToString(), Field.Store.NO));
w.UpdateDocument(new Term("id", "" + id), doc);
docsInSegment.IncrementAndGet();
// TODO: fix this test
if (SlowFileExists(dir, "_0_1.del") || SlowFileExists(dir, "_0_1.liv"))
{
if (VERBOSE)
{
Console.WriteLine("TEST: deletes created @ id=" + id);
}
break;
}
id++;
}
closing.Set(true);
Assert.IsTrue(sawAfterFlush.Get());
w.Dispose();
dir.Dispose();
}
public void testConcurrentAddWidgets()
{
WidgetBuilder widgetBuilder =
new WidgetBuilder(new Class[] { });
//www.it - ebooks.infoGood Comments
String text = "'''bold text'''";
ParentWidget parent = null;
AtomicBoolean failFlag = new AtomicBoolean();
failFlag.set(false);
//This is our best attempt to get a race condition
//by creating large number of threads.
for (int i = 0; i < 25000; i++)
{
WidgetBuilderThread widgetBuilderThread =
new WidgetBuilderThread(widgetBuilder, text, parent, failFlag);
Thread thread = new Thread(new ThreadStart(assertEquals));
thread.Start();
}
assertEquals(false, failFlag.get());
}
private void startRecordRig()
{
recordedRigs = new List<Body[]>();
recordedRigTimePoints = new List<int>();
rigDetected = new AtomicBoolean(false);
try
{
XmlWriterSettings ws = new XmlWriterSettings();
ws.Indent = true;
rigWriter = XmlWriter.Create(tempRigFileName, ws);
rigWriter.WriteStartDocument();
rigWriter.WriteStartElement("BodyDataSequence");
}
catch (Exception e)
{
Console.WriteLine(e);
}
}
public void SetValue_ManyConcurrentThreads_OnlyOneSucceeds_Test()
{
_atomicBoolean = new AtomicBoolean(false);
int totalSucceedingThreads = 0;
Parallel.For(1, 11, (state) =>
{
System.Console.WriteLine(String.Format("Executing SetValue on thread {0}",
Thread.CurrentThread.ManagedThreadId));
if (!_atomicBoolean.SetValue(true))
{
System.Console.WriteLine(String.Format("Thread {0} was successful in calling SetValue.",
Thread.CurrentThread.ManagedThreadId));
Interlocked.Increment(ref totalSucceedingThreads);
}
});
//Assert
_atomicBoolean.Value.Should().BeTrue();
totalSucceedingThreads.Should().Be(1);
}
public Subscriber(AtomicBoolean running, Subscription subscription)
{
Running = running;
Subscription = subscription;
}
public virtual void TestAccquireReleaseRace()
{
DocumentsWriterStallControl ctrl = new DocumentsWriterStallControl();
ctrl.UpdateStalled(false);
AtomicBoolean stop = new AtomicBoolean(false);
AtomicBoolean checkPoint = new AtomicBoolean(true);
int numStallers = AtLeast(1);
int numReleasers = AtLeast(1);
int numWaiters = AtLeast(1);
var sync = new Synchronizer(numStallers + numReleasers, numStallers + numReleasers + numWaiters);
var threads = new ThreadClass[numReleasers + numStallers + numWaiters];
IList<Exception> exceptions = new SynchronizedCollection<Exception>();
for (int i = 0; i < numReleasers; i++)
{
threads[i] = new Updater(stop, checkPoint, ctrl, sync, true, exceptions);
}
for (int i = numReleasers; i < numReleasers + numStallers; i++)
{
threads[i] = new Updater(stop, checkPoint, ctrl, sync, false, exceptions);
}
for (int i = numReleasers + numStallers; i < numReleasers + numStallers + numWaiters; i++)
{
threads[i] = new Waiter(stop, checkPoint, ctrl, sync, exceptions);
}
Start(threads);
int iters = AtLeast(10000);
float checkPointProbability = TEST_NIGHTLY ? 0.5f : 0.1f;
for (int i = 0; i < iters; i++)
{
if (checkPoint.Get())
{
Assert.IsTrue(sync.UpdateJoin.@await(new TimeSpan(0, 0, 0, 10)), "timed out waiting for update threads - deadlock?");
if (exceptions.Count > 0)
{
foreach (Exception throwable in exceptions)
{
Console.WriteLine(throwable.ToString());
Console.Write(throwable.StackTrace);
}
Assert.Fail("got exceptions in threads");
}
if (ctrl.HasBlocked() && ctrl.Healthy)
{
AssertState(numReleasers, numStallers, numWaiters, threads, ctrl);
}
checkPoint.Set(false);
sync.Waiter.countDown();
sync.LeftCheckpoint.@await();
}
Assert.IsFalse(checkPoint.Get());
Assert.AreEqual(0, sync.Waiter.Remaining);
if (checkPointProbability >= (float)Random().NextDouble())
{
sync.Reset(numStallers + numReleasers, numStallers + numReleasers + numWaiters);
checkPoint.Set(true);
}
}
if (!checkPoint.Get())
{
sync.Reset(numStallers + numReleasers, numStallers + numReleasers + numWaiters);
checkPoint.Set(true);
}
Assert.IsTrue(sync.UpdateJoin.@await(new TimeSpan(0, 0, 0, 10)));
AssertState(numReleasers, numStallers, numWaiters, threads, ctrl);
checkPoint.Set(false);
stop.Set(true);
sync.Waiter.countDown();
sync.LeftCheckpoint.@await();
for (int i = 0; i < threads.Length; i++)
{
ctrl.UpdateStalled(false);
threads[i].Join(2000);
if (threads[i].IsAlive && threads[i] is Waiter)
{
if (threads[i].State == ThreadState.WaitSleepJoin)
{
Assert.Fail("waiter is not released - anyThreadsStalled: " + ctrl.AnyStalledThreads());
}
}
}
}
public BlockingCollection(IProducerConsumerCollection <T> underlyingColl, int upperBound)
{
this.underlyingColl = underlyingColl;
this.upperBound = upperBound;
this.isComplete = new AtomicBoolean();
}
public ThreadAnonymousInnerClassHelper(TestFieldCache outerInstance, IFieldCache cache, AtomicBoolean failed, AtomicInt32 iters, int NUM_ITER, Barrier restart)
{
this.OuterInstance = outerInstance;
this.Cache = cache;
this.Failed = failed;
this.Iters = iters;
this.NUM_ITER = NUM_ITER;
this.Restart = restart;
}
public ThreadAnonymousInnerClassHelper(TestTaxonomyCombined outerInstance, Lucene.Net.Facet.Taxonomy.FacetLabel abPath, int abOrd, int abYoungChildBase1, int abYoungChildBase2, int retry, DirectoryTaxonomyReader tr, AtomicBoolean stop, Exception[] error, int[] retrieval)
: base("Child Arrays Verifier")
{
this.outerInstance = outerInstance;
this.abPath = abPath;
this.abOrd = abOrd;
this.abYoungChildBase1 = abYoungChildBase1;
this.abYoungChildBase2 = abYoungChildBase2;
this.retry = retry;
this.tr = tr;
this.stop = stop;
this.error = error;
this.retrieval = retrieval;
}
public Subscriber(AtomicBoolean running, Subscription subscription)
{
Running = running;
Subscription = subscription;
}
public _Runnable_564(TestLazyPersistFiles _enclosing, Path path1, int Seed, AtomicBoolean
testFailed, CountDownLatch latch)
{
this._enclosing = _enclosing;
this.path1 = path1;
this.Seed = Seed;
this.testFailed = testFailed;
this.latch = latch;
}
/// <summary>
/// TBD
/// </summary>
/// <param name="settings">TBD</param>
/// <param name="haveShutdown">TBD</param>
public StreamSupervisor(ActorMaterializerSettings settings, AtomicBoolean haveShutdown)
{
Settings = settings;
HaveShutdown = haveShutdown;
}
/// <summary> /// TBD /// </summary> /// <param name="settings">TBD</param> /// <param name="haveShutdown">TBD</param> /// <returns>TBD</returns> public static Props Props(ActorMaterializerSettings settings, AtomicBoolean haveShutdown) => Actor.Props.Create(() => new StreamSupervisor(settings, haveShutdown)).WithDeploy(Deploy.Local);
/// <summary>
/// TBD
/// </summary>
/// <param name="system">TBD</param>
/// <param name="settings">TBD</param>
/// <param name="dispatchers">TBD</param>
/// <param name="supervisor">TBD</param>
/// <param name="haveShutDown">TBD</param>
/// <param name="flowNames">TBD</param>
/// <returns>TBD</returns>
public ActorMaterializerImpl(ActorSystem system, ActorMaterializerSettings settings, Dispatchers dispatchers, IActorRef supervisor, AtomicBoolean haveShutDown, EnumerableActorName flowNames)
{
_system = system;
_settings = settings;
_dispatchers = dispatchers;
_supervisor = supervisor;
_haveShutDown = haveShutDown;
_flowNames = flowNames;
_executionContext = new Lazy <MessageDispatcher>(() => _dispatchers.Lookup(_settings.Dispatcher == Deploy.NoDispatcherGiven
? Dispatchers.DefaultDispatcherId
: _settings.Dispatcher));
if (_settings.IsFuzzingMode && !_system.Settings.Config.HasPath("akka.stream.secret-test-fuzzing-warning-disable"))
{
Logger.Warning("Fuzzing mode is enabled on this system. If you see this warning on your production system then set 'akka.materializer.debug.fuzzing-mode' to off.");
}
}
public static void Main()
{
ComputerSpecifications.Dump();
var reporter = new RateReporter(1000, PrintRate);
var rateReporterHandler = SamplesUtil.RateReporterHandler(reporter);
var context = new Aeron.Context();
var running = new AtomicBoolean(true);
var reportThread = new Thread(reporter.Run);
var subscribeThread = new Thread(subscription => SamplesUtil.SubscriberLoop(rateReporterHandler, FragmentCountLimit, running)((Subscription)subscription));
using (var aeron = Aeron.Connect(context))
using (var publication = aeron.AddPublication(Channel, StreamID))
using (var subscription = aeron.AddSubscription(Channel, StreamID))
using (var byteBuffer = BufferUtil.AllocateDirectAligned(MessageLength, BitUtil.CACHE_LINE_LENGTH))
using (var buffer = new UnsafeBuffer(byteBuffer))
{
reportThread.Start();
subscribeThread.Start(subscription);
do
{
Console.WriteLine("Streaming {0:G} messages of size {1:G} bytes to {2} on stream Id {3}", NumberOfMessages, MessageLength, Channel, StreamID);
_printingActive = true;
long backPressureCount = 0;
for (long i = 0; i < NumberOfMessages; i++)
{
buffer.PutLong(0, i);
OfferIdleStrategy.Reset();
while (publication.Offer(buffer, 0, buffer.Capacity) < 0)
{
OfferIdleStrategy.Idle();
backPressureCount++;
}
}
Console.WriteLine("Done streaming. backPressureRatio=" + (double)backPressureCount / NumberOfMessages);
if (0 < LingerTimeoutMs)
{
Console.WriteLine("Lingering for " + LingerTimeoutMs + " milliseconds...");
Thread.Sleep((int)LingerTimeoutMs);
}
_printingActive = false;
} while (Console.ReadLine() != "x");
reporter.Halt();
running.Set(false);
if (!subscribeThread.Join(5000))
{
Console.WriteLine("Warning: not all tasks completed promptly");
}
}
}
public RateReporter(AtomicBoolean running, Subscriber subscriber)
{
Running = running;
Subscriber = subscriber;
_stopwatch = Stopwatch.StartNew();
}
public IndexWriterAnonymousInnerClassHelper(TestIndexWriterDelete outerInstance, Directory dir, IndexWriterConfig setReaderPooling, AtomicInteger docsInSegment, AtomicBoolean closing, AtomicBoolean sawAfterFlush)
: base(dir, setReaderPooling)
{
this.OuterInstance = outerInstance;
this.DocsInSegment = docsInSegment;
this.Closing = closing;
this.SawAfterFlush = sawAfterFlush;
}
public RateReporter(AtomicBoolean running, Subscriber subscriber)
{
Running = running;
Subscriber = subscriber;
_stopwatch = Stopwatch.StartNew();
}
public ThreadAnonymousInnerClassHelper(TestIndexWriterWithThreads outerInstance, BaseDirectoryWrapper d, AtomicReference <IndexWriter> writerRef, LineFileDocs docs, int iters, AtomicBoolean failed, ReentrantLock rollbackLock, ReentrantLock commitLock)
{
this.OuterInstance = outerInstance;
this.d = d;
this.WriterRef = writerRef;
this.Docs = docs;
this.Iters = iters;
this.Failed = failed;
this.RollbackLock = rollbackLock;
this.CommitLock = commitLock;
}
public Publisher(AtomicBoolean running, Publication publication)
{
Running = running;
Publication = publication;
}
/// <exception cref="System.Exception"/>
public virtual void TestDSShell(bool haveDomain)
{
string[] args = new string[] { "--jar", AppmasterJar, "--num_containers", "2", "--shell_command"
, Shell.Windows ? "dir" : "ls", "--master_memory", "512", "--master_vcores", "2"
, "--container_memory", "128", "--container_vcores", "1" };
if (haveDomain)
{
string[] domainArgs = new string[] { "--domain", "TEST_DOMAIN", "--view_acls", "reader_user reader_group"
, "--modify_acls", "writer_user writer_group", "--create" };
IList <string> argsList = new AList <string>(Arrays.AsList(args));
Sharpen.Collections.AddAll(argsList, Arrays.AsList(domainArgs));
args = Sharpen.Collections.ToArray(argsList, new string[argsList.Count]);
}
Log.Info("Initializing DS Client");
Client client = new Client(new Configuration(yarnCluster.GetConfig()));
bool initSuccess = client.Init(args);
NUnit.Framework.Assert.IsTrue(initSuccess);
Log.Info("Running DS Client");
AtomicBoolean result = new AtomicBoolean(false);
Sharpen.Thread t = new _Thread_194(result, client);
t.Start();
YarnClient yarnClient = YarnClient.CreateYarnClient();
yarnClient.Init(new Configuration(yarnCluster.GetConfig()));
yarnClient.Start();
string hostName = NetUtils.GetHostname();
bool verified = false;
string errorMessage = string.Empty;
while (!verified)
{
IList <ApplicationReport> apps = yarnClient.GetApplications();
if (apps.Count == 0)
{
Sharpen.Thread.Sleep(10);
continue;
}
ApplicationReport appReport = apps[0];
if (appReport.GetHost().Equals("N/A"))
{
Sharpen.Thread.Sleep(10);
continue;
}
errorMessage = "Expected host name to start with '" + hostName + "', was '" + appReport
.GetHost() + "'. Expected rpc port to be '-1', was '" + appReport.GetRpcPort() +
"'.";
if (CheckHostname(appReport.GetHost()) && appReport.GetRpcPort() == -1)
{
verified = true;
}
if (appReport.GetYarnApplicationState() == YarnApplicationState.Finished)
{
break;
}
}
NUnit.Framework.Assert.IsTrue(errorMessage, verified);
t.Join();
Log.Info("Client run completed. Result=" + result);
NUnit.Framework.Assert.IsTrue(result.Get());
TimelineDomain domain = null;
if (haveDomain)
{
domain = yarnCluster.GetApplicationHistoryServer().GetTimelineStore().GetDomain("TEST_DOMAIN"
);
NUnit.Framework.Assert.IsNotNull(domain);
NUnit.Framework.Assert.AreEqual("reader_user reader_group", domain.GetReaders());
NUnit.Framework.Assert.AreEqual("writer_user writer_group", domain.GetWriters());
}
TimelineEntities entitiesAttempts = yarnCluster.GetApplicationHistoryServer().GetTimelineStore
().GetEntities(ApplicationMaster.DSEntity.DsAppAttempt.ToString(), null, null, null
, null, null, null, null, null, null);
NUnit.Framework.Assert.IsNotNull(entitiesAttempts);
NUnit.Framework.Assert.AreEqual(1, entitiesAttempts.GetEntities().Count);
NUnit.Framework.Assert.AreEqual(2, entitiesAttempts.GetEntities()[0].GetEvents().
Count);
NUnit.Framework.Assert.AreEqual(entitiesAttempts.GetEntities()[0].GetEntityType()
.ToString(), ApplicationMaster.DSEntity.DsAppAttempt.ToString());
if (haveDomain)
{
NUnit.Framework.Assert.AreEqual(domain.GetId(), entitiesAttempts.GetEntities()[0]
.GetDomainId());
}
else
{
NUnit.Framework.Assert.AreEqual("DEFAULT", entitiesAttempts.GetEntities()[0].GetDomainId
());
}
TimelineEntities entities = yarnCluster.GetApplicationHistoryServer().GetTimelineStore
().GetEntities(ApplicationMaster.DSEntity.DsContainer.ToString(), null, null, null
, null, null, null, null, null, null);
NUnit.Framework.Assert.IsNotNull(entities);
NUnit.Framework.Assert.AreEqual(2, entities.GetEntities().Count);
NUnit.Framework.Assert.AreEqual(entities.GetEntities()[0].GetEntityType().ToString
(), ApplicationMaster.DSEntity.DsContainer.ToString());
if (haveDomain)
{
NUnit.Framework.Assert.AreEqual(domain.GetId(), entities.GetEntities()[0].GetDomainId
());
}
else
{
NUnit.Framework.Assert.AreEqual("DEFAULT", entities.GetEntities()[0].GetDomainId(
));
}
}
public RefreshListenerAnonymousClass(TestControlledRealTimeReopenThread outerInstance, AtomicBoolean afterRefreshCalled)
{
this.outerInstance = outerInstance;
this.afterRefreshCalled = afterRefreshCalled;
}
public ConcurrentMergeSchedulerAnonymousClass(TestConcurrentMergeScheduler outerInstance, int maxMergeCount, CountdownEvent enoughMergesWaiting, AtomicInt32 runningMergeCount, AtomicBoolean failed)
{
this.outerInstance = outerInstance;
this.maxMergeCount = maxMergeCount;
this.enoughMergesWaiting = enoughMergesWaiting;
this.runningMergeCount = runningMergeCount;
this.failed = failed;
}
public BlockingCollection(IProducerConsumerCollection <T> collection, int boundedCapacity)
{
this.underlyingColl = collection;
this.upperBound = boundedCapacity;
this.isComplete = new AtomicBoolean();
}
public ThreadAnonymousInnerClassHelper(AtomicBoolean stop, SearcherTaxonomyManager mgr)
{
this.stop = stop;
this.mgr = mgr;
}
/// <summary>
/// TBD
/// </summary>
/// <param name="breaker">TBD</param>
public HalfOpen(CircuitBreaker breaker)
: base(breaker.CallTimeout, 0)
{
_breaker = breaker;
_lock = new AtomicBoolean();
}
public ThreadAnonymousClass2(TestControlledRealTimeReopenThread outerInstance, long lastGen, ControlledRealTimeReopenThread <IndexSearcher> thread, AtomicBoolean finished)
{
this.outerInstance = outerInstance;
this.lastGen = lastGen;
this.thread = thread;
this.finished = finished;
}
public DeviceClientWrapper(DeviceClient deviceClient)
{
this.underlyingDeviceClient = Preconditions.CheckNotNull(deviceClient);
this.isActive = new AtomicBoolean(true);
}
private static ThreadStart NewInfiniteWaitingRunnable(AtomicBoolean exitCondition)
{
return(() => Predicates.awaitForever(() => Thread.CurrentThread.Interrupted || exitCondition.get(), 500, MILLISECONDS));
}
public _FilterChain_104(AtomicBoolean invoked)
{
this.invoked = invoked;
}
public ThreadAnonymousInnerClassHelper(TestDocValuesIndexing outerInstance, IndexWriter w, CountdownEvent startingGun, AtomicBoolean hitExc, Document doc)
{
this.OuterInstance = outerInstance;
this.w = w;
this.StartingGun = startingGun;
this.HitExc = hitExc;
this.Doc = doc;
}
public virtual void TestMixedTypesDifferentThreads()
{
Directory dir = NewDirectory();
IndexWriter w = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())));
CountdownEvent startingGun = new CountdownEvent(1);
AtomicBoolean hitExc = new AtomicBoolean();
ThreadClass[] threads = new ThreadClass[3];
for (int i = 0; i < 3; i++)
{
Field field;
if (i == 0)
{
field = new SortedDocValuesField("foo", new BytesRef("hello"));
}
else if (i == 1)
{
field = new NumericDocValuesField("foo", 0);
}
else
{
field = new BinaryDocValuesField("foo", new BytesRef("bazz"));
}
Document doc = new Document();
doc.Add(field);
threads[i] = new ThreadAnonymousInnerClassHelper(this, w, startingGun, hitExc, doc);
threads[i].Start();
}
startingGun.Signal();
foreach (ThreadClass t in threads)
{
t.Join();
}
Assert.IsTrue(hitExc.Get());
w.Dispose();
dir.Dispose();
}
public Waiter(AtomicBoolean stop, AtomicBoolean checkPoint, DocumentsWriterStallControl ctrl, Synchronizer sync, IList<Exception> exceptions)
: base("waiter")
{
this.Stop = stop;
this.CheckPoint = checkPoint;
this.Ctrl = ctrl;
this.Sync = sync;
this.Exceptions = exceptions;
}
/// <summary>
/// Construct a var in a given namespace with a given name.
/// </summary>
/// <param name="ns">The namespace.</param>
/// <param name="sym">The var.</param>
internal Var(Namespace ns, Symbol sym)
{
_ns = ns;
_sym = sym;
_threadBound = new AtomicBoolean(false);
_root = new Unbound(this);
setMeta(PersistentHashMap.EMPTY);
}
public virtual void TestThreadStarvationNoDeleteNRTReader()
{
IndexWriterConfig conf = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random));
conf.SetMergePolicy(Random.NextBoolean() ? NoMergePolicy.COMPOUND_FILES : NoMergePolicy.NO_COMPOUND_FILES);
Directory d = NewDirectory();
CountdownEvent latch = new CountdownEvent(1);
CountdownEvent signal = new CountdownEvent(1);
LatchedIndexWriter _writer = new LatchedIndexWriter(d, conf, latch, signal);
TrackingIndexWriter writer = new TrackingIndexWriter(_writer);
SearcherManager manager = new SearcherManager(_writer, false, null);
Document doc = new Document();
doc.Add(NewTextField("test", "test", Field.Store.YES));
writer.AddDocument(doc);
manager.MaybeRefresh();
var t = new ThreadAnonymousClass(this, latch, signal, writer, manager);
t.Start();
_writer.waitAfterUpdate = true; // wait in addDocument to let some reopens go through
long lastGen = writer.UpdateDocument(new Term("foo", "bar"), doc); // once this returns the doc is already reflected in the last reopen
assertFalse(manager.IsSearcherCurrent()); // false since there is a delete in the queue
IndexSearcher searcher = manager.Acquire();
try
{
assertEquals(2, searcher.IndexReader.NumDocs);
}
finally
{
manager.Release(searcher);
}
ControlledRealTimeReopenThread <IndexSearcher> thread = new ControlledRealTimeReopenThread <IndexSearcher>(writer, manager, 0.01, 0.01);
thread.Start(); // start reopening
if (Verbose)
{
Console.WriteLine("waiting now for generation " + lastGen);
}
AtomicBoolean finished = new AtomicBoolean(false);
var waiter = new ThreadAnonymousClass2(this, lastGen, thread, finished);
waiter.Start();
manager.MaybeRefresh();
waiter.Join(1000);
if (!finished)
{
waiter.Interrupt();
fail("thread deadlocked on waitForGeneration");
}
thread.Dispose();
thread.Join();
IOUtils.Dispose(manager, _writer, d);
}
public ConcurrentMergeSchedulerAnonymousInnerClassHelper(TestConcurrentMergeScheduler outerInstance, int maxMergeCount, CountdownEvent enoughMergesWaiting, AtomicInteger runningMergeCount, AtomicBoolean failed)
{
this.OuterInstance = outerInstance;
this.MaxMergeCount = maxMergeCount;
this.EnoughMergesWaiting = enoughMergesWaiting;
this.RunningMergeCount = runningMergeCount;
this.Failed = failed;
}
public Publisher(AtomicBoolean running, Publication publication)
{
Running = running;
Publication = publication;
}
public _Thread_194(AtomicBoolean result, Client client)
{
this.result = result;
this.client = client;
}
private void AssertConsistentYoungestChild(FacetLabel abPath, int abOrd, int abYoungChildBase1, int abYoungChildBase2, int retry, int numCategories)
{
var indexDir = new SlowRAMDirectory(-1, null); // no slowness for intialization
var tw = new DirectoryTaxonomyWriter(indexDir);
tw.AddCategory(new FacetLabel("a", "0"));
tw.AddCategory(abPath);
tw.Commit();
var tr = new DirectoryTaxonomyReader(indexDir);
for (int i = 0; i < numCategories; i++)
{
var cp = new FacetLabel("a", "b", Convert.ToString(i));
tw.AddCategory(cp);
Assert.AreEqual(TaxonomyReader.INVALID_ORDINAL, tr.GetOrdinal(cp), "Ordinal of " + cp + " must be invalid until Taxonomy Reader was refreshed");
}
tw.Dispose();
var stop = new AtomicBoolean(false);
Exception[] error = new Exception[] { null };
int[] retrieval = new int[] { 0 };
var thread = new ThreadAnonymousInnerClassHelper(this, abPath, abOrd, abYoungChildBase1, abYoungChildBase2, retry, tr, stop, error, retrieval);
thread.Start();
indexDir.SleepMillis = 1; // some delay for refresh
var newTaxoReader = TaxonomyReader.OpenIfChanged(tr);
if (newTaxoReader != null)
{
newTaxoReader.Dispose();
}
stop.Set(true);
thread.Join();
Assert.Null(error[0], "Unexpcted exception at retry " + retry + " retrieval " + retrieval[0] + ": \n" + stackTraceStr(error[0]));
tr.Dispose();
}
public IndexerThread(IndexWriter w, FacetsConfig config, ITaxonomyWriter tw, ReferenceManager <SearcherAndTaxonomy> mgr, int ordLimit, AtomicBoolean stop)
{
this.w = w;
this.config = config;
this.tw = tw;
this.mgr = mgr;
this.ordLimit = ordLimit;
this.stop = stop;
}
public async Task CommittablePartitionedSource_Should_handle_exceptions_in_stream_without_commit_failures()
{
var partitionsCount = 3;
var topic = CreateTopic(1);
var group = CreateGroup(1);
var totalMessages = 100;
var exceptionTriggered = new AtomicBoolean(false);
var allTopicPartitions = Enumerable.Range(0, partitionsCount).Select(i => new TopicPartition(topic, i)).ToList();
var consumerSettings = CreateConsumerSettings <string>(group).WithStopTimeout(TimeSpan.FromSeconds(2));
var createdSubSources = new ConcurrentSet <TopicPartition>();
var commitFailures = new ConcurrentSet <(TopicPartition, Exception)>();
await ProduceStrings(i => new TopicPartition(topic, i % partitionsCount), Enumerable.Range(1, totalMessages), ProducerSettings);
var control = KafkaConsumer.CommittablePartitionedSource(consumerSettings, Subscriptions.Topics(topic))
.GroupBy(partitionsCount, tuple => tuple.Item1)
.SelectAsync(6, tuple =>
{
var(topicPartition, source) = tuple;
createdSubSources.TryAdd(topicPartition);
return(source
.Log($"Subsource for partition #{topicPartition.Partition.Value}", m => m.Record.Value)
.SelectAsync(3, async message =>
{
// fail on first partition; otherwise delay slightly and emit
if (topicPartition.Partition.Value == 0)
{
Log.Debug($"Failing {topicPartition} source");
exceptionTriggered.GetAndSet(true);
throw new Exception("FAIL");
}
else
{
await Task.Delay(50);
}
return message;
})
.Log($"Subsource {topicPartition} pre commit")
.SelectAsync(1, async message =>
{
try
{
await message.CommitableOffset.Commit();
}
catch (Exception ex)
{
Log.Error("Commit failure: " + ex);
commitFailures.TryAdd((topicPartition, ex));
}
return message;
})
.Scan(0, (c, _) => c + 1)
.RunWith(Sink.Last <int>(), Materializer)
.ContinueWith(t =>
{
Log.Info($"sub-source for {topicPartition} completed: Received {t.Result} messages in total.");
return t.Result;
}));
})
public TestRecoveryRecoveryCallback(AtomicBoolean boolean)
{
Boolean = boolean;
}
public ThreadAnonymousInnerClassHelper(TestDocValuesIndexing outerInstance, IndexWriter w, CountdownEvent startingGun, AtomicBoolean hitExc, Document doc)
{
this.OuterInstance = outerInstance;
this.w = w;
this.StartingGun = startingGun;
this.HitExc = hitExc;
this.Doc = doc;
}
public virtual void TestNrt()
{
Store.Directory dir = NewDirectory();
Store.Directory taxoDir = NewDirectory();
IndexWriterConfig iwc = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random));
// Don't allow tiny maxBufferedDocs; it can make this
// test too slow:
iwc.SetMaxBufferedDocs(Math.Max(500, iwc.MaxBufferedDocs));
// MockRandom/AlcololicMergePolicy are too slow:
TieredMergePolicy tmp = new TieredMergePolicy();
tmp.FloorSegmentMB = .001;
iwc.SetMergePolicy(tmp);
IndexWriter w = new IndexWriter(dir, iwc);
var tw = new DirectoryTaxonomyWriter(taxoDir);
FacetsConfig config = new FacetsConfig();
config.SetMultiValued("field", true);
AtomicBoolean stop = new AtomicBoolean();
// How many unique facets to index before stopping:
int ordLimit = TestNightly ? 100000 : 6000;
var indexer = new IndexerThread(w, config, tw, null, ordLimit, stop);
var mgr = new SearcherTaxonomyManager(w, true, null, tw);
var reopener = new ThreadAnonymousInnerClassHelper(this, stop, mgr);
reopener.Name = "reopener";
reopener.Start();
indexer.Name = "indexer";
indexer.Start();
try
{
while (!stop)
{
SearcherAndTaxonomy pair = mgr.Acquire();
try
{
//System.out.println("search maxOrd=" + pair.taxonomyReader.getSize());
FacetsCollector sfc = new FacetsCollector();
pair.Searcher.Search(new MatchAllDocsQuery(), sfc);
Facets facets = GetTaxonomyFacetCounts(pair.TaxonomyReader, config, sfc);
FacetResult result = facets.GetTopChildren(10, "field");
if (pair.Searcher.IndexReader.NumDocs > 0)
{
//System.out.println(pair.taxonomyReader.getSize());
Assert.True(result.ChildCount > 0);
Assert.True(result.LabelValues.Length > 0);
}
//if (VERBOSE) {
//System.out.println("TEST: facets=" + FacetTestUtils.toString(results.get(0)));
//}
}
finally
{
mgr.Release(pair);
}
}
}
finally
{
indexer.Join();
reopener.Join();
}
if (Verbose)
{
Console.WriteLine("TEST: now stop");
}
IOUtils.Dispose(mgr, tw, w, taxoDir, dir);
}
public virtual void TestMaxMergeCount()
{
Directory dir = NewDirectory();
IndexWriterConfig iwc = new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()));
int maxMergeCount = TestUtil.NextInt(Random(), 1, 5);
int maxMergeThreads = TestUtil.NextInt(Random(), 1, maxMergeCount);
CountdownEvent enoughMergesWaiting = new CountdownEvent(maxMergeCount);
AtomicInteger runningMergeCount = new AtomicInteger(0);
AtomicBoolean failed = new AtomicBoolean();
if (VERBOSE)
{
Console.WriteLine("TEST: maxMergeCount=" + maxMergeCount + " maxMergeThreads=" + maxMergeThreads);
}
ConcurrentMergeScheduler cms = new ConcurrentMergeSchedulerAnonymousInnerClassHelper(this, maxMergeCount, enoughMergesWaiting, runningMergeCount, failed);
cms.SetMaxMergesAndThreads(maxMergeCount, maxMergeThreads);
iwc.SetMergeScheduler(cms);
iwc.SetMaxBufferedDocs(2);
TieredMergePolicy tmp = new TieredMergePolicy();
iwc.SetMergePolicy(tmp);
tmp.MaxMergeAtOnce = 2;
tmp.SegmentsPerTier = 2;
IndexWriter w = new IndexWriter(dir, iwc);
Document doc = new Document();
doc.Add(NewField("field", "field", TextField.TYPE_NOT_STORED));
while (enoughMergesWaiting.CurrentCount != 0 && !failed.Get())
{
for (int i = 0; i < 10; i++)
{
w.AddDocument(doc);
}
}
w.Dispose(false);
dir.Dispose();
}
public ThreadAnonymousInnerClassHelper(TestSearcherTaxonomyManager outerInstance, AtomicBoolean stop, Lucene.Net.Facet.Taxonomy.SearcherTaxonomyManager mgr)
{
this.outerInstance = outerInstance;
this.stop = stop;
this.mgr = mgr;
}
public bool Equals(AtomicBoolean rhs)
{
return this.flag == rhs.flag;
}
public virtual void Test_Directory() // LUCENENET specific - name collides with property of LuceneTestCase
{
Store.Directory indexDir = NewDirectory();
Store.Directory taxoDir = NewDirectory();
IndexWriter w = new IndexWriter(indexDir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random)));
var tw = new DirectoryTaxonomyWriter(taxoDir);
// first empty commit
w.Commit();
tw.Commit();
var mgr = new SearcherTaxonomyManager(indexDir, taxoDir, null);
FacetsConfig config = new FacetsConfig();
config.SetMultiValued("field", true);
AtomicBoolean stop = new AtomicBoolean();
// How many unique facets to index before stopping:
int ordLimit = TestNightly ? 100000 : 6000;
var indexer = new IndexerThread(w, config, tw, mgr, ordLimit, stop);
indexer.Start();
try
{
while (!stop)
{
SearcherAndTaxonomy pair = mgr.Acquire();
try
{
//System.out.println("search maxOrd=" + pair.taxonomyReader.getSize());
FacetsCollector sfc = new FacetsCollector();
pair.Searcher.Search(new MatchAllDocsQuery(), sfc);
Facets facets = GetTaxonomyFacetCounts(pair.TaxonomyReader, config, sfc);
FacetResult result = facets.GetTopChildren(10, "field");
if (pair.Searcher.IndexReader.NumDocs > 0)
{
//System.out.println(pair.taxonomyReader.getSize());
Assert.True(result.ChildCount > 0);
Assert.True(result.LabelValues.Length > 0);
}
//if (VERBOSE) {
//System.out.println("TEST: facets=" + FacetTestUtils.toString(results.get(0)));
//}
}
finally
{
mgr.Release(pair);
}
}
}
finally
{
indexer.Join();
}
if (Verbose)
{
Console.WriteLine("TEST: now stop");
}
IOUtils.Dispose(mgr, tw, w, taxoDir, indexDir);
}
