public virtual void TestAddResourceRefAddResourceConcurrency()
{
StartEmptyStore();
string key = "key1";
string fileName = "foo.jar";
string user = "******";
ApplicationId id = CreateAppId(1, 1L);
// add the resource and add the resource ref at the same time
ExecutorService exec = Executors.NewFixedThreadPool(2);
CountDownLatch start = new CountDownLatch(1);
Callable <string> addKeyTask = new _Callable_240(this, start, key, fileName);
Callable <string> addAppIdTask = new _Callable_246(this, start, key, id, user);
Future <string> addAppIdFuture = exec.Submit(addAppIdTask);
Future <string> addKeyFuture = exec.Submit(addKeyTask);
// start them at the same time
start.CountDown();
// get the results
string addKeyResult = addKeyFuture.Get();
string addAppIdResult = addAppIdFuture.Get();
NUnit.Framework.Assert.AreEqual(fileName, addKeyResult);
System.Console.Out.WriteLine("addAppId() result: " + addAppIdResult);
// it may be null or the fileName depending on the timing
NUnit.Framework.Assert.IsTrue(addAppIdResult == null || addAppIdResult.Equals(fileName
));
exec.Shutdown();
}
/// <summary>
/// Test that we get an AsynchronousCloseException when the DomainSocket
/// we're using is closed during a read or write operation.
/// </summary>
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
private void TestAsyncCloseDuringIO(bool closeDuringWrite)
{
string TestPath = new FilePath(sockDir.GetDir(), "testAsyncCloseDuringIO(" + closeDuringWrite
+ ")").GetAbsolutePath();
DomainSocket serv = DomainSocket.BindAndListen(TestPath);
ExecutorService exeServ = Executors.NewFixedThreadPool(2);
Callable <Void> serverCallable = new _Callable_180(serv, closeDuringWrite);
// The server just continues either writing or reading until someone
// asynchronously closes the client's socket. At that point, all our
// reads return EOF, and writes get a socket error.
Future <Void> serverFuture = exeServ.Submit(serverCallable);
DomainSocket clientConn = DomainSocket.Connect(serv.GetPath());
Callable <Void> clientCallable = new _Callable_213(closeDuringWrite, clientConn);
// The client writes or reads until another thread
// asynchronously closes the socket. At that point, we should
// get ClosedChannelException, or possibly its subclass
// AsynchronousCloseException.
Future <Void> clientFuture = exeServ.Submit(clientCallable);
Thread.Sleep(500);
clientConn.Close();
serv.Close();
clientFuture.Get(2, TimeUnit.Minutes);
serverFuture.Get(2, TimeUnit.Minutes);
}
public virtual void Handle(ContainersLauncherEvent @event)
{
// TODO: ContainersLauncher launches containers one by one!!
Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container container
= @event.GetContainer();
ContainerId containerId = container.GetContainerId();
switch (@event.GetType())
{
case ContainersLauncherEventType.LaunchContainer:
{
Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application
app = context.GetApplications()[containerId.GetApplicationAttemptId().GetApplicationId
()];
ContainerLaunch launch = new ContainerLaunch(context, GetConfig(), dispatcher, exec
, app, @event.GetContainer(), dirsHandler, containerManager);
containerLauncher.Submit(launch);
running[containerId] = launch;
break;
}
case ContainersLauncherEventType.RecoverContainer:
{
app = context.GetApplications()[containerId.GetApplicationAttemptId().GetApplicationId
()];
launch = new RecoveredContainerLaunch(context, GetConfig(), dispatcher, exec, app
, @event.GetContainer(), dirsHandler, containerManager);
containerLauncher.Submit(launch);
running[containerId] = launch;
break;
}
case ContainersLauncherEventType.CleanupContainer:
{
ContainerLaunch launcher = Sharpen.Collections.Remove(running, containerId);
if (launcher == null)
{
// Container not launched. So nothing needs to be done.
return;
}
// Cleanup a container whether it is running/killed/completed, so that
// no sub-processes are alive.
try
{
launcher.CleanupContainer();
}
catch (IOException)
{
Log.Warn("Got exception while cleaning container " + containerId + ". Ignoring.");
}
break;
}
}
}
internal virtual Future <byte[]> SubmitAllocate()
{
count.IncrementAndGet();
Future <byte[]> f = pool.Submit(new _Callable_438(this));
lock (arrays)
{
arrays.AddItem(f);
}
return(f);
}
public virtual void TestCounter()
{
long countResetTimePeriodMs = 200L;
ByteArrayManager.Counter c = new ByteArrayManager.Counter(countResetTimePeriodMs);
int n = DFSUtil.GetRandom().Next(512) + 512;
IList <Future <int> > futures = new AList <Future <int> >(n);
ExecutorService pool = Executors.NewFixedThreadPool(32);
try
{
// increment
for (int i = 0; i < n; i++)
{
futures.AddItem(pool.Submit(new _Callable_82(c)));
}
// sort and wait for the futures
futures.Sort(Cmp);
}
finally
{
pool.Shutdown();
}
// check futures
NUnit.Framework.Assert.AreEqual(n, futures.Count);
for (int i_1 = 0; i_1 < n; i_1++)
{
NUnit.Framework.Assert.AreEqual(i_1 + 1, futures[i_1].Get());
}
NUnit.Framework.Assert.AreEqual(n, c.GetCount());
// test auto-reset
Sharpen.Thread.Sleep(countResetTimePeriodMs + 100);
NUnit.Framework.Assert.AreEqual(1, c.Increment());
}
internal virtual Future <int> Submit(byte[] array)
{
Future <int> f = pool.Submit(new _Callable_287(this, array));
furtures.AddItem(f);
return(f);
}
internal virtual Future <byte[]> Submit(int arrayLength)
{
Future <byte[]> f = pool.Submit(new _Callable_255(this, arrayLength));
futures.AddItem(f);
return(f);
}
/// <summary>Run a set of tasks and waits for them to complete.</summary>
/// <exception cref="System.Exception"/>
private void RunTasks(IList <LocalJobRunner.Job.RunnableWithThrowable> runnables,
ExecutorService service, string taskType)
{
// Start populating the executor with work units.
// They may begin running immediately (in other threads).
foreach (Runnable r in runnables)
{
service.Submit(r);
}
try
{
service.Shutdown();
// Instructs queue to drain.
// Wait for tasks to finish; do not use a time-based timeout.
// (See http://bugs.sun.com/bugdatabase/view_bug.do?bug_id=6179024)
LocalJobRunner.Log.Info("Waiting for " + taskType + " tasks");
service.AwaitTermination(long.MaxValue, TimeUnit.Nanoseconds);
}
catch (Exception ie)
{
// Cancel all threads.
service.ShutdownNow();
throw;
}
LocalJobRunner.Log.Info(taskType + " task executor complete.");
// After waiting for the tasks to complete, if any of these
// have thrown an exception, rethrow it now in the main thread context.
foreach (LocalJobRunner.Job.RunnableWithThrowable r_1 in runnables)
{
if (r_1.storedException != null)
{
throw new Exception(r_1.storedException);
}
}
}
public virtual void TestAddResourceConcurrency()
{
StartEmptyStore();
string key = "key1";
int count = 5;
ExecutorService exec = Executors.NewFixedThreadPool(count);
IList <Future <string> > futures = new AList <Future <string> >(count);
CountDownLatch start = new CountDownLatch(1);
for (int i = 0; i < count; i++)
{
string fileName = "foo-" + i + ".jar";
Callable <string> task = new _Callable_129(this, start, key, fileName);
futures.AddItem(exec.Submit(task));
}
// start them all at the same time
start.CountDown();
// check the result; they should all agree with the value
ICollection <string> results = new HashSet <string>();
foreach (Future <string> future in futures)
{
results.AddItem(future.Get());
}
NUnit.Framework.Assert.AreSame(1, results.Count);
exec.Shutdown();
}
public virtual void TestStopThreads()
{
DelegationTokenManager tm1 = null;
string connectString = zkServer.GetConnectString();
// let's make the update interval short and the shutdown interval
// comparatively longer, so if the update thread runs after shutdown,
// it will cause an error.
long updateIntervalSeconds = 1;
long shutdownTimeoutMillis = updateIntervalSeconds * 1000 * 5;
Configuration conf = GetSecretConf(connectString);
conf.SetLong(DelegationTokenManager.UpdateInterval, updateIntervalSeconds);
conf.SetLong(DelegationTokenManager.RemovalScanInterval, updateIntervalSeconds);
conf.SetLong(DelegationTokenManager.RenewInterval, updateIntervalSeconds);
conf.SetLong(ZKDelegationTokenSecretManager.ZkDtsmZkShutdownTimeout, shutdownTimeoutMillis
);
tm1 = new DelegationTokenManager(conf, new Text("foo"));
tm1.Init();
Org.Apache.Hadoop.Security.Token.Token <DelegationTokenIdentifier> token = (Org.Apache.Hadoop.Security.Token.Token
<DelegationTokenIdentifier>)tm1.CreateToken(UserGroupInformation.GetCurrentUser(
), "foo");
NUnit.Framework.Assert.IsNotNull(token);
AbstractDelegationTokenSecretManager sm = tm1.GetDelegationTokenSecretManager();
ZKDelegationTokenSecretManager zksm = (ZKDelegationTokenSecretManager)sm;
ExecutorService es = zksm.GetListenerThreadPool();
es.Submit(new _Callable_300(shutdownTimeoutMillis));
// force this to be shutdownNow
tm1.Destroy();
}
/// <exception cref="System.Exception"/>
private void SingleRun()
{
FsUrlStreamHandlerFactory factory = new FsUrlStreamHandlerFactory();
Random random = new Random();
ExecutorService executor = Executors.NewFixedThreadPool(Threads);
AList <Future <object> > futures = new AList <Future <object> >(Tasks);
for (int i = 0; i < Tasks; i++)
{
int aux = i;
futures.AddItem(executor.Submit(new _Runnable_55(aux, random, factory)));
}
executor.Shutdown();
try
{
executor.AwaitTermination(Timeout, TimeUnit.Seconds);
executor.ShutdownNow();
}
catch (Exception)
{
}
// pass
// check for exceptions
foreach (Future future in futures)
{
if (!future.IsDone())
{
break;
}
// timed out
future.Get();
}
}
/// <exception cref="System.Exception"/>
public virtual void TestUniqueDestinationPath()
{
Configuration conf = new Configuration();
FileContext files = FileContext.GetLocalFSFileContext(conf);
Path basedir = files.MakeQualified(new Path("target", typeof(TestFSDownload).Name
));
files.Mkdir(basedir, null, true);
conf.SetStrings(typeof(TestFSDownload).FullName, basedir.ToString());
ExecutorService singleThreadedExec = Executors.NewSingleThreadExecutor();
LocalDirAllocator dirs = new LocalDirAllocator(typeof(TestFSDownload).FullName);
Path destPath = dirs.GetLocalPathForWrite(basedir.ToString(), conf);
destPath = new Path(destPath, System.Convert.ToString(uniqueNumberGenerator.IncrementAndGet
()));
Path p = new Path(basedir, "dir" + 0 + ".jar");
LocalResourceVisibility vis = LocalResourceVisibility.Private;
LocalResource rsrc = CreateJar(files, p, vis);
FSDownload fsd = new FSDownload(files, UserGroupInformation.GetCurrentUser(), conf
, destPath, rsrc);
Future <Path> rPath = singleThreadedExec.Submit(fsd);
singleThreadedExec.Shutdown();
while (!singleThreadedExec.AwaitTermination(1000, TimeUnit.Milliseconds))
{
}
NUnit.Framework.Assert.IsTrue(rPath.IsDone());
// Now FSDownload will not create a random directory to localize the
// resource. Therefore the final localizedPath for the resource should be
// destination directory (passed as an argument) + file name.
NUnit.Framework.Assert.AreEqual(destPath, rPath.Get().GetParent());
}
public virtual void ConcurrentRepack()
{
//
// leave the syncPoint in broken state so any awaiting
// threads and any threads that call await in the future get
// the BrokenBarrierException
//
CyclicBarrier syncPoint = new CyclicBarrier(1);
RevBlob a = tr.Blob("a");
tr.LightweightTag("t", a);
ExecutorService pool = Executors.NewFixedThreadPool(2);
try
{
_T187790690 repack1 = new _T187790690(this)
{
syncPoint = syncPoint
};
//_T187790690 repack2 = new _T187790690(this) { syncPoint = syncPoint };
Future <int> result1 = pool.Submit(repack1);
//Future<int> result2 = pool.Submit(repack2);
NUnit.Framework.Assert.AreEqual(0, result1.Get()); // + result2.Get());
}
finally
{
pool.Shutdown();
pool.AwaitTermination(long.MaxValue, TimeUnit.SECONDS);
}
}
/// <exception cref="System.Exception"/>
public virtual void TestThreadSafety()
{
// Test for thread safety by starting multiple threads that mutate the same
// StartupProgress instance in various ways. We expect no internal
// corruption of data structures and no lost updates on counter increments.
int numThreads = 100;
// Data tables used by each thread to determine values to pass to APIs.
Phase[] phases = new Phase[] { Phase.LoadingFsimage, Phase.LoadingFsimage, Phase.
LoadingEdits, Phase.LoadingEdits };
Step[] steps = new Step[] { new Step(StepType.Inodes), new Step(StepType.DelegationKeys
), new Step(StepType.Inodes), new Step(StepType.DelegationKeys) };
string[] files = new string[] { "file1", "file1", "file2", "file2" };
long[] sizes = new long[] { 1000L, 1000L, 2000L, 2000L };
long[] totals = new long[] { 10000L, 20000L, 30000L, 40000L };
ExecutorService exec = Executors.NewFixedThreadPool(numThreads);
try
{
for (int i = 0; i < numThreads; ++i)
{
Phase phase = phases[i % phases.Length];
Step step = steps[i % steps.Length];
string file = files[i % files.Length];
long size = sizes[i % sizes.Length];
long total = totals[i % totals.Length];
exec.Submit(new _Callable_369(this, phase, file, size, step, total));
}
}
finally
{
exec.Shutdown();
NUnit.Framework.Assert.IsTrue(exec.AwaitTermination(10000L, TimeUnit.Milliseconds
));
}
StartupProgressView view = startupProgress.CreateView();
NUnit.Framework.Assert.IsNotNull(view);
NUnit.Framework.Assert.AreEqual("file1", view.GetFile(Phase.LoadingFsimage));
NUnit.Framework.Assert.AreEqual(1000L, view.GetSize(Phase.LoadingFsimage));
NUnit.Framework.Assert.AreEqual(10000L, view.GetTotal(Phase.LoadingFsimage, new Step
(StepType.Inodes)));
NUnit.Framework.Assert.AreEqual(2500L, view.GetCount(Phase.LoadingFsimage, new Step
(StepType.Inodes)));
NUnit.Framework.Assert.AreEqual(20000L, view.GetTotal(Phase.LoadingFsimage, new Step
(StepType.DelegationKeys)));
NUnit.Framework.Assert.AreEqual(2500L, view.GetCount(Phase.LoadingFsimage, new Step
(StepType.DelegationKeys)));
NUnit.Framework.Assert.AreEqual("file2", view.GetFile(Phase.LoadingEdits));
NUnit.Framework.Assert.AreEqual(2000L, view.GetSize(Phase.LoadingEdits));
NUnit.Framework.Assert.AreEqual(30000L, view.GetTotal(Phase.LoadingEdits, new Step
(StepType.Inodes)));
NUnit.Framework.Assert.AreEqual(2500L, view.GetCount(Phase.LoadingEdits, new Step
(StepType.Inodes)));
NUnit.Framework.Assert.AreEqual(40000L, view.GetTotal(Phase.LoadingEdits, new Step
(StepType.DelegationKeys)));
NUnit.Framework.Assert.AreEqual(2500L, view.GetCount(Phase.LoadingEdits, new Step
(StepType.DelegationKeys)));
}
/// <summary>Verify the RPC server can shutdown properly when callQueue is full.</summary>
/// <exception cref="System.Exception"/>
public virtual void TestRPCServerShutdown()
{
int numClients = 3;
IList <Future <Void> > res = new AList <Future <Void> >();
ExecutorService executorService = Executors.NewFixedThreadPool(numClients);
Configuration conf = new Configuration();
conf.SetInt(CommonConfigurationKeys.IpcClientConnectMaxRetriesKey, 0);
Server server = new RPC.Builder(conf).SetProtocol(typeof(TestRPC.TestProtocol)).SetInstance
(new TestRPC.TestImpl()).SetBindAddress(Address).SetPort(0).SetQueueSizePerHandler
(1).SetNumHandlers(1).SetVerbose(true).Build();
server.Start();
TestRPC.TestProtocol proxy = RPC.GetProxy <TestRPC.TestProtocol>(TestRPC.TestProtocol
.versionID, NetUtils.GetConnectAddress(server), conf);
try
{
// start a sleep RPC call to consume the only handler thread.
// Start another sleep RPC call to make callQueue full.
// Start another sleep RPC call to make reader thread block on CallQueue.
for (int i = 0; i < numClients; i++)
{
res.AddItem(executorService.Submit(new _Callable_1046(proxy)));
}
while (server.GetCallQueueLen() != 1 && CountThreads(typeof(CallQueueManager).FullName
) != 1 && CountThreads(typeof(TestRPC.TestProtocol).FullName) != 1)
{
Thread.Sleep(100);
}
}
finally
{
try
{
server.Stop();
Assert.Equal("Not enough clients", numClients, res.Count);
foreach (Future <Void> f in res)
{
try
{
f.Get();
NUnit.Framework.Assert.Fail("Future get should not return");
}
catch (ExecutionException e)
{
Assert.True("Unexpected exception: " + e, e.InnerException is IOException
);
Log.Info("Expected exception", e.InnerException);
}
}
}
finally
{
RPC.StopProxy(proxy);
executorService.Shutdown();
}
}
}
public void SubmitReturnsFutureThatReturnsResultOfCompletedAction()
{
var runnable = MockRepository.GenerateStub <IRunnable>();
IFuture <Void> future = ExecutorService.Submit(runnable);
future.GetResult();
runnable.AssertWasCalled(c => c.Run());
JoinPool(ExecutorService);
}
public void SubmitExecutesTheRunnable()
{
var future = ExecutorService.Submit(_runnable);
future.GetResult();
Assert.IsTrue(future.IsDone);
_runnable.AssertWasCalled(x => x.Run());
JoinPool(ExecutorService);
}
public void SubmitExecutesTheAction()
{
var future = ExecutorService.Submit(_action);
future.GetResult();
Assert.IsTrue(future.IsDone);
_action.AssertWasCalled(t => t());
JoinPool(ExecutorService);
}
public long Run(ThroughputSessionContext sessionContext)
{
var latch = new ManualResetEvent(false);
_stepThreeFunctionHandler.Reset(latch, _stepThreeBatchProcessor.Sequence.Value + _iterations);
var processorTask1 = _executor.Submit(_stepOneBatchProcessor);
var processorTask2 = _executor.Submit(_stepTwoBatchProcessor);
var processorTask3 = _executor.Submit(_stepThreeBatchProcessor);
_stepOneBatchProcessor.WaitUntilStarted(TimeSpan.FromSeconds(5));
_stepTwoBatchProcessor.WaitUntilStarted(TimeSpan.FromSeconds(5));
_stepThreeBatchProcessor.WaitUntilStarted(TimeSpan.FromSeconds(5));
sessionContext.Start();
var ringBuffer = _ringBuffer;
var operandTwo = _operandTwoInitialValue;
for (long i = 0; i < _iterations; i++)
{
var sequence = ringBuffer.Next();
var @event =
ringBuffer[sequence];
@event.OperandOne = i;
@event.OperandTwo = operandTwo--;
ringBuffer.Publish(sequence);
}
latch.WaitOne();
sessionContext.Stop();
_stepOneBatchProcessor.Halt();
_stepTwoBatchProcessor.Halt();
_stepThreeBatchProcessor.Halt();
Task.WaitAll(processorTask1, processorTask2, processorTask3);
PerfTestUtil.FailIfNot(_expectedResult, _stepThreeFunctionHandler.StepThreeCounter);
return(_iterations);
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="Sharpen.URISyntaxException"/>
/// <exception cref="System.Exception"/>
public virtual void TestDownloadBadPublic()
{
Configuration conf = new Configuration();
conf.Set(CommonConfigurationKeys.FsPermissionsUmaskKey, "077");
FileContext files = FileContext.GetLocalFSFileContext(conf);
Path basedir = files.MakeQualified(new Path("target", typeof(TestFSDownload).Name
));
files.Mkdir(basedir, null, true);
conf.SetStrings(typeof(TestFSDownload).FullName, basedir.ToString());
IDictionary <LocalResource, LocalResourceVisibility> rsrcVis = new Dictionary <LocalResource
, LocalResourceVisibility>();
Random rand = new Random();
long sharedSeed = rand.NextLong();
rand.SetSeed(sharedSeed);
System.Console.Out.WriteLine("SEED: " + sharedSeed);
IDictionary <LocalResource, Future <Path> > pending = new Dictionary <LocalResource,
Future <Path> >();
ExecutorService exec = Executors.NewSingleThreadExecutor();
LocalDirAllocator dirs = new LocalDirAllocator(typeof(TestFSDownload).FullName);
int size = 512;
LocalResourceVisibility vis = LocalResourceVisibility.Public;
Path path = new Path(basedir, "test-file");
LocalResource rsrc = CreateFile(files, path, size, rand, vis);
rsrcVis[rsrc] = vis;
Path destPath = dirs.GetLocalPathForWrite(basedir.ToString(), size, conf);
destPath = new Path(destPath, System.Convert.ToString(uniqueNumberGenerator.IncrementAndGet
()));
FSDownload fsd = new FSDownload(files, UserGroupInformation.GetCurrentUser(), conf
, destPath, rsrc);
pending[rsrc] = exec.Submit(fsd);
exec.Shutdown();
while (!exec.AwaitTermination(1000, TimeUnit.Milliseconds))
{
}
NUnit.Framework.Assert.IsTrue(pending[rsrc].IsDone());
try
{
foreach (KeyValuePair <LocalResource, Future <Path> > p in pending)
{
p.Value.Get();
NUnit.Framework.Assert.Fail("We localized a file that is not public.");
}
}
catch (ExecutionException e)
{
NUnit.Framework.Assert.IsTrue(e.InnerException is IOException);
}
}
/// <exception cref="System.Exception"/>
public virtual void TestMultiThreadedCompressorPool()
{
int iterations = 4;
ExecutorService threadpool = Executors.NewFixedThreadPool(3);
LinkedBlockingDeque <Compressor> queue = new LinkedBlockingDeque <Compressor>(2 * iterations
);
Callable <bool> consumer = new _Callable_110(queue);
Callable <bool> producer = new _Callable_119(this, queue);
for (int i = 0; i < iterations; i++)
{
threadpool.Submit(consumer);
threadpool.Submit(producer);
}
// wait for completion
threadpool.Shutdown();
threadpool.AwaitTermination(1000, TimeUnit.Seconds);
Assert.Equal(LeaseCountErr, 0, CodecPool.GetLeasedCompressorsCount
(codec));
}
public virtual void TestConcurrentAccess()
{
conf.Set(FairSchedulerConfiguration.AssignMultiple, "false");
resourceManager = new MockRM(conf);
resourceManager.Start();
scheduler = (FairScheduler)resourceManager.GetResourceScheduler();
string queueName = "root.queue1";
FSLeafQueue schedulable = scheduler.GetQueueManager().GetLeafQueue(queueName, true
);
ApplicationAttemptId applicationAttemptId = CreateAppAttemptId(1, 1);
RMContext rmContext = resourceManager.GetRMContext();
FSAppAttempt app = new FSAppAttempt(scheduler, applicationAttemptId, "user1", schedulable
, null, rmContext);
// this needs to be in sync with the number of runnables declared below
int testThreads = 2;
IList <Runnable> runnables = new AList <Runnable>();
// add applications to modify the list
runnables.AddItem(new _Runnable_257(schedulable, app));
// iterate over the list a couple of times in a different thread
runnables.AddItem(new _Runnable_267(schedulable));
IList <Exception> exceptions = Sharpen.Collections.SynchronizedList(new AList <Exception
>());
ExecutorService threadPool = Executors.NewFixedThreadPool(testThreads);
try
{
CountDownLatch allExecutorThreadsReady = new CountDownLatch(testThreads);
CountDownLatch startBlocker = new CountDownLatch(1);
CountDownLatch allDone = new CountDownLatch(testThreads);
foreach (Runnable submittedTestRunnable in runnables)
{
threadPool.Submit(new _Runnable_287(allExecutorThreadsReady, startBlocker, submittedTestRunnable
, exceptions, allDone));
}
// wait until all threads are ready
allExecutorThreadsReady.Await();
// start all test runners
startBlocker.CountDown();
int testTimeout = 2;
NUnit.Framework.Assert.IsTrue("Timeout waiting for more than " + testTimeout + " seconds"
, allDone.Await(testTimeout, TimeUnit.Seconds));
}
catch (Exception ie)
{
exceptions.AddItem(ie);
}
finally
{
threadPool.ShutdownNow();
}
NUnit.Framework.Assert.IsTrue("Test failed with exception(s)" + exceptions, exceptions
.IsEmpty());
}
public virtual void ConcurrentPackRefs_onlyOneWritesPackedRefs()
{
RevBlob a = tr.Blob("a");
tr.LightweightTag("t", a);
CyclicBarrier syncPoint = new CyclicBarrier(2);
Callable <int> packRefs = new _Callable_131(this, syncPoint);
ExecutorService pool = Executors.NewFixedThreadPool(2);
try
{
Future <int> p1 = pool.Submit(packRefs);
Future <int> p2 = pool.Submit(packRefs);
NUnit.Framework.Assert.AreEqual(1, p1.Get() + p2.Get());
}
finally
{
pool.Shutdown();
pool.AwaitTermination(long.MaxValue, TimeUnit.SECONDS);
}
}
/// <exception cref="System.Exception"/>
private void TestConcurrentCommitTaskWithSubDir(int version)
{
Job job = Job.GetInstance();
FileOutputFormat.SetOutputPath(job, outDir);
Configuration conf = job.GetConfiguration();
conf.Set(MRJobConfig.TaskAttemptId, attempt);
conf.SetInt(FileOutputCommitter.FileoutputcommitterAlgorithmVersion, version);
conf.SetClass("fs.file.impl", typeof(TestFileOutputCommitter.RLFS), typeof(FileSystem
));
FileSystem.CloseAll();
JobContext jContext = new JobContextImpl(conf, taskID.GetJobID());
FileOutputCommitter amCommitter = new FileOutputCommitter(outDir, jContext);
amCommitter.SetupJob(jContext);
TaskAttemptContext[] taCtx = new TaskAttemptContextImpl[2];
taCtx[0] = new TaskAttemptContextImpl(conf, taskID);
taCtx[1] = new TaskAttemptContextImpl(conf, taskID1);
TextOutputFormat[] tof = new TextOutputFormat[2];
for (int i = 0; i < tof.Length; i++)
{
tof[i] = new _TextOutputFormat_508(this);
}
ExecutorService executor = Executors.NewFixedThreadPool(2);
try
{
for (int i_1 = 0; i_1 < taCtx.Length; i_1++)
{
int taskIdx = i_1;
executor.Submit(new _Callable_524(this, tof, taskIdx, taCtx));
}
}
finally
{
executor.Shutdown();
while (!executor.AwaitTermination(1, TimeUnit.Seconds))
{
Log.Info("Awaiting thread termination!");
}
}
amCommitter.CommitJob(jContext);
RawLocalFileSystem lfs = new RawLocalFileSystem();
lfs.SetConf(conf);
NUnit.Framework.Assert.IsFalse("Must not end up with sub_dir/sub_dir", lfs.Exists
(new Path(OutSubDir, SubDir)));
// validate output
ValidateContent(OutSubDir);
FileUtil.FullyDelete(new FilePath(outDir.ToString()));
}
public virtual void TestParallelThreadsWithDifferentLocales()
{
CyclicBarrier barrier = new CyclicBarrier(2);
// wait for the other thread to set its locale
ExecutorService pool = Executors.NewFixedThreadPool(2);
try
{
Future <TranslationBundle> root = pool.Submit(new _T879158014(this, NLS.ROOT_LOCALE,
barrier));
Future <TranslationBundle> german = pool.Submit(new _T879158014(this, Sharpen.Extensions.GetGermanCulture(),
barrier));
NUnit.Framework.Assert.AreEqual(NLS.ROOT_LOCALE, root.Get().EffectiveLocale());
NUnit.Framework.Assert.AreEqual(Sharpen.Extensions.GetGermanCulture(), german.Get
().EffectiveLocale());
}
finally
{
pool.Shutdown();
pool.AwaitTermination(long.MaxValue, TimeUnit.SECONDS);
}
}
/// <summary>
/// Test that if one thread is blocking in a read or write operation, another
/// thread can close the socket and stop the accept.
/// </summary>
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
public virtual void TestSocketAcceptAndClose()
{
string TestPath = new FilePath(sockDir.GetDir(), "test_sock_accept_and_close").GetAbsolutePath
();
DomainSocket serv = DomainSocket.BindAndListen(TestPath);
ExecutorService exeServ = Executors.NewSingleThreadExecutor();
Callable <Void> callable = new _Callable_149(serv);
Future <Void> future = exeServ.Submit(callable);
Thread.Sleep(500);
serv.Close();
future.Get(2, TimeUnit.Minutes);
}
public virtual void Handle(SharedCacheUploadEvent @event)
{
if (enabled)
{
IDictionary <LocalResourceRequest, Path> resources = @event.GetResources();
foreach (KeyValuePair <LocalResourceRequest, Path> e in resources)
{
SharedCacheUploader uploader = new SharedCacheUploader(e.Key, e.Value, @event.GetUser
(), GetConfig(), scmClient, fs, localFs);
// fire off an upload task
uploaderPool.Submit(uploader);
}
}
}
public virtual void PackRefsWhileRefUpdated_refUpdateSucceeds()
{
RevBlob a = tr.Blob("a");
tr.LightweightTag("t", a);
RevBlob b = tr.Blob("b");
CyclicBarrier refUpdateLockedRef = new CyclicBarrier(2);
CyclicBarrier packRefsDone = new CyclicBarrier(2);
ExecutorService pool = Executors.NewFixedThreadPool(2);
try
{
Future <RefUpdate.Result> result = pool.Submit(new _Callable_185(this, b, refUpdateLockedRef
, packRefsDone));
pool.Submit <object>(new _Callable_210(this, refUpdateLockedRef, packRefsDone));
NUnit.Framework.Assert.AreEqual(result.Get(), RefUpdate.Result.FORCED);
}
finally
{
pool.ShutdownNow();
pool.AwaitTermination(long.MaxValue, TimeUnit.SECONDS);
}
NUnit.Framework.Assert.AreEqual(repo.GetRef("refs/tags/t").GetObjectId(), b);
}
/// <summary>Test that we get a read result of -1 on EOF.</summary>
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
public virtual void TestSocketReadEof()
{
string TestPath = new FilePath(sockDir.GetDir(), "testSocketReadEof").GetAbsolutePath
();
DomainSocket serv = DomainSocket.BindAndListen(TestPath);
ExecutorService exeServ = Executors.NewSingleThreadExecutor();
Callable <Void> callable = new _Callable_109(serv);
Future <Void> future = exeServ.Submit(callable);
DomainSocket conn = DomainSocket.Connect(serv.GetPath());
Thread.Sleep(50);
conn.Close();
serv.Close();
future.Get(2, TimeUnit.Minutes);
}
/// <exception cref="System.Exception"/>
/// <exception cref="ExecutionException"/>
public virtual void TestOperations(int input, int numberOfThreads, int pause, bool
success, bool attemptedBefore, Server.Call call)
{
int failureOutput = input + 1;
ExecutorService executorService = Executors.NewFixedThreadPool(numberOfThreads);
IList <Future <int> > list = new AList <Future <int> >();
for (int i = 0; i < numberOfThreads; i++)
{
Callable <int> worker = new _Callable_178(call, pause, input, failureOutput, success
);
Future <int> submit = executorService.Submit(worker);
list.AddItem(submit);
}
Assert.Equal(numberOfThreads, list.Count);
foreach (Future <int> future in list)
{
if (success)
{
Assert.Equal(input, future.Get());
}
else
{
Assert.Equal(failureOutput, future.Get());
}
}
if (success)
{
// If the operation was successful, all the subsequent operations
// by other threads should be retries. Operation count should be 1.
int retries = numberOfThreads + (attemptedBefore ? 0 : -1);
Assert.Equal(1, testServer.operationCount.Get());
Assert.Equal(retries, testServer.retryCount.Get());
}
else
{
// If the operation failed, all the subsequent operations
// should execute once more, hence the retry count should be 0 and
// operation count should be the number of tries
int opCount = numberOfThreads + (attemptedBefore ? 1 : 0);
Assert.Equal(opCount, testServer.operationCount.Get());
Assert.Equal(0, testServer.retryCount.Get());
}
}
