/// <summary>Run a set of tasks and waits for them to complete.</summary>
/// <exception cref="System.Exception"/>
private void RunTasks(IList <LocalJobRunner.Job.RunnableWithThrowable> runnables,
ExecutorService service, string taskType)
{
// Start populating the executor with work units.
// They may begin running immediately (in other threads).
foreach (Runnable r in runnables)
{
service.Submit(r);
}
try
{
service.Shutdown();
// Instructs queue to drain.
// Wait for tasks to finish; do not use a time-based timeout.
// (See http://bugs.sun.com/bugdatabase/view_bug.do?bug_id=6179024)
LocalJobRunner.Log.Info("Waiting for " + taskType + " tasks");
service.AwaitTermination(long.MaxValue, TimeUnit.Nanoseconds);
}
catch (Exception ie)
{
// Cancel all threads.
service.ShutdownNow();
throw;
}
LocalJobRunner.Log.Info(taskType + " task executor complete.");
// After waiting for the tasks to complete, if any of these
// have thrown an exception, rethrow it now in the main thread context.
foreach (LocalJobRunner.Job.RunnableWithThrowable r_1 in runnables)
{
if (r_1.storedException != null)
{
throw new Exception(r_1.storedException);
}
}
}
/// <exception cref="System.Exception"/>
private void SingleRun()
{
FsUrlStreamHandlerFactory factory = new FsUrlStreamHandlerFactory();
Random random = new Random();
ExecutorService executor = Executors.NewFixedThreadPool(Threads);
AList <Future <object> > futures = new AList <Future <object> >(Tasks);
for (int i = 0; i < Tasks; i++)
{
int aux = i;
futures.AddItem(executor.Submit(new _Runnable_55(aux, random, factory)));
}
executor.Shutdown();
try
{
executor.AwaitTermination(Timeout, TimeUnit.Seconds);
executor.ShutdownNow();
}
catch (Exception)
{
}
// pass
// check for exceptions
foreach (Future future in futures)
{
if (!future.IsDone())
{
break;
}
// timed out
future.Get();
}
}
/// <exception cref="System.IO.IOException"></exception>
public virtual void Close()
{
closed.Set(true);
executorService.ShutdownNow();
if (httpclient is IDisposable)
{
((IDisposable)httpclient).Close();
}
}
public virtual void TestConcurrentAccess()
{
conf.Set(FairSchedulerConfiguration.AssignMultiple, "false");
resourceManager = new MockRM(conf);
resourceManager.Start();
scheduler = (FairScheduler)resourceManager.GetResourceScheduler();
string queueName = "root.queue1";
FSLeafQueue schedulable = scheduler.GetQueueManager().GetLeafQueue(queueName, true
);
ApplicationAttemptId applicationAttemptId = CreateAppAttemptId(1, 1);
RMContext rmContext = resourceManager.GetRMContext();
FSAppAttempt app = new FSAppAttempt(scheduler, applicationAttemptId, "user1", schedulable
, null, rmContext);
// this needs to be in sync with the number of runnables declared below
int testThreads = 2;
IList <Runnable> runnables = new AList <Runnable>();
// add applications to modify the list
runnables.AddItem(new _Runnable_257(schedulable, app));
// iterate over the list a couple of times in a different thread
runnables.AddItem(new _Runnable_267(schedulable));
IList <Exception> exceptions = Sharpen.Collections.SynchronizedList(new AList <Exception
>());
ExecutorService threadPool = Executors.NewFixedThreadPool(testThreads);
try
{
CountDownLatch allExecutorThreadsReady = new CountDownLatch(testThreads);
CountDownLatch startBlocker = new CountDownLatch(1);
CountDownLatch allDone = new CountDownLatch(testThreads);
foreach (Runnable submittedTestRunnable in runnables)
{
threadPool.Submit(new _Runnable_287(allExecutorThreadsReady, startBlocker, submittedTestRunnable
, exceptions, allDone));
}
// wait until all threads are ready
allExecutorThreadsReady.Await();
// start all test runners
startBlocker.CountDown();
int testTimeout = 2;
NUnit.Framework.Assert.IsTrue("Timeout waiting for more than " + testTimeout + " seconds"
, allDone.Await(testTimeout, TimeUnit.Seconds));
}
catch (Exception ie)
{
exceptions.AddItem(ie);
}
finally
{
threadPool.ShutdownNow();
}
NUnit.Framework.Assert.IsTrue("Test failed with exception(s)" + exceptions, exceptions
.IsEmpty());
}
private void StopAggregators()
{
threadPool.Shutdown();
// if recovery on restart is supported then leave outstanding aggregations
// to the next restart
bool shouldAbort = context.GetNMStateStore().CanRecover() && !context.GetDecommissioned
();
// politely ask to finish
foreach (AppLogAggregator aggregator in appLogAggregators.Values)
{
if (shouldAbort)
{
aggregator.AbortLogAggregation();
}
else
{
aggregator.FinishLogAggregation();
}
}
while (!threadPool.IsTerminated())
{
// wait for all threads to finish
foreach (ApplicationId appId in appLogAggregators.Keys)
{
Log.Info("Waiting for aggregation to complete for " + appId);
}
try
{
if (!threadPool.AwaitTermination(30, TimeUnit.Seconds))
{
threadPool.ShutdownNow();
}
}
catch (Exception)
{
// send interrupt to hurry them along
Log.Warn("Aggregation stop interrupted!");
break;
}
}
foreach (ApplicationId appId_1 in appLogAggregators.Keys)
{
Log.Warn("Some logs may not have been aggregated for " + appId_1);
}
}
/// <param name="service">
///
/// <see cref="ExecutorService">to be shutdown</see>
/// </param>
/// <param name="timeoutInMs">
/// time to wait for
/// <see cref="ExecutorService.AwaitTermination(long, TimeUnit)"/>
/// calls in milli seconds.
/// </param>
/// <returns>
/// <tt>true</tt> if the service is terminated,
/// <tt>false</tt> otherwise
/// </returns>
/// <exception cref="System.Exception"/>
public static bool ShutdownExecutorService(ExecutorService service, long timeoutInMs
)
{
if (service == null)
{
return(true);
}
service.Shutdown();
if (!service.AwaitTermination(timeoutInMs, TimeUnit.Milliseconds))
{
service.ShutdownNow();
return(service.AwaitTermination(timeoutInMs, TimeUnit.Milliseconds));
}
else
{
return(true);
}
}
public virtual void PackRefsWhileRefUpdated_refUpdateSucceeds()
{
RevBlob a = tr.Blob("a");
tr.LightweightTag("t", a);
RevBlob b = tr.Blob("b");
CyclicBarrier refUpdateLockedRef = new CyclicBarrier(2);
CyclicBarrier packRefsDone = new CyclicBarrier(2);
ExecutorService pool = Executors.NewFixedThreadPool(2);
try
{
Future <RefUpdate.Result> result = pool.Submit(new _Callable_185(this, b, refUpdateLockedRef
, packRefsDone));
pool.Submit <object>(new _Callable_210(this, refUpdateLockedRef, packRefsDone));
NUnit.Framework.Assert.AreEqual(result.Get(), RefUpdate.Result.FORCED);
}
finally
{
pool.ShutdownNow();
pool.AwaitTermination(long.MaxValue, TimeUnit.SECONDS);
}
NUnit.Framework.Assert.AreEqual(repo.GetRef("refs/tags/t").GetObjectId(), b);
}
public override List <Runnable> ShutdownNow()
{
return(e.ShutdownNow());
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
public virtual int RunLocalization(IPEndPoint nmAddr)
{
// load credentials
InitDirs(conf, user, appId, lfs, localDirs);
Credentials creds = new Credentials();
DataInputStream credFile = null;
try
{
// assume credentials in cwd
// TODO: Fix
Path tokenPath = new Path(string.Format(TokenFileNameFmt, localizerId));
credFile = lfs.Open(tokenPath);
creds.ReadTokenStorageStream(credFile);
// Explicitly deleting token file.
lfs.Delete(tokenPath, false);
}
finally
{
if (credFile != null)
{
credFile.Close();
}
}
// create localizer context
UserGroupInformation remoteUser = UserGroupInformation.CreateRemoteUser(user);
remoteUser.AddToken(creds.GetToken(LocalizerTokenIdentifier.Kind));
LocalizationProtocol nodeManager = remoteUser.DoAs(new _PrivilegedAction_151(this
, nmAddr));
// create user context
UserGroupInformation ugi = UserGroupInformation.CreateRemoteUser(user);
foreach (Org.Apache.Hadoop.Security.Token.Token <TokenIdentifier> token in creds.GetAllTokens
())
{
ugi.AddToken(token);
}
ExecutorService exec = null;
try
{
exec = CreateDownloadThreadPool();
CompletionService <Path> ecs = CreateCompletionService(exec);
LocalizeFiles(nodeManager, ecs, ugi);
return(0);
}
catch (Exception e)
{
// Print traces to stdout so that they can be logged by the NM address
// space.
Sharpen.Runtime.PrintStackTrace(e, System.Console.Out);
return(-1);
}
finally
{
try
{
if (exec != null)
{
exec.ShutdownNow();
}
LocalDirAllocator.RemoveContext(appCacheDirContextName);
}
finally
{
CloseFileSystems(ugi);
}
}
}
/// <exception cref="System.Exception"/>
protected override void ServiceStop()
{
containerLauncher.ShutdownNow();
base.ServiceStop();
}
public override void StopThreads()
{
base.StopThreads();
try
{
if (tokenCache != null)
{
tokenCache.Close();
}
}
catch (Exception e)
{
Log.Error("Could not stop Delegation Token Cache", e);
}
try
{
if (delTokSeqCounter != null)
{
delTokSeqCounter.Close();
}
}
catch (Exception e)
{
Log.Error("Could not stop Delegation Token Counter", e);
}
try
{
if (keyIdSeqCounter != null)
{
keyIdSeqCounter.Close();
}
}
catch (Exception e)
{
Log.Error("Could not stop Key Id Counter", e);
}
try
{
if (keyCache != null)
{
keyCache.Close();
}
}
catch (Exception e)
{
Log.Error("Could not stop KeyCache", e);
}
try
{
if (!isExternalClient && (zkClient != null))
{
zkClient.Close();
}
}
catch (Exception e)
{
Log.Error("Could not stop Curator Framework", e);
}
if (listenerThreadPool != null)
{
listenerThreadPool.Shutdown();
try
{
// wait for existing tasks to terminate
if (!listenerThreadPool.AwaitTermination(shutdownTimeout, TimeUnit.Milliseconds))
{
Log.Error("Forcing Listener threadPool to shutdown !!");
listenerThreadPool.ShutdownNow();
}
}
catch (Exception)
{
listenerThreadPool.ShutdownNow();
Thread.CurrentThread().Interrupt();
}
}
}