/// <exception cref="System.Exception"/>
public virtual void TestDSShellWithNodeLabelExpression()
{
InitializeNodeLabels();
// Start NMContainerMonitor
TestDistributedShellWithNodeLabels.NMContainerMonitor mon = new TestDistributedShellWithNodeLabels.NMContainerMonitor
(this);
Sharpen.Thread t = new Sharpen.Thread(mon);
t.Start();
// Submit a job which will sleep for 60 sec
string[] args = new string[] { "--jar", TestDistributedShell.AppmasterJar, "--num_containers"
, "4", "--shell_command", "sleep", "--shell_args", "15", "--master_memory", "512"
, "--master_vcores", "2", "--container_memory", "128", "--container_vcores", "1"
, "--node_label_expression", "x" };
Log.Info("Initializing DS Client");
Client client = new Client(new Configuration(distShellTest.yarnCluster.GetConfig(
)));
bool initSuccess = client.Init(args);
NUnit.Framework.Assert.IsTrue(initSuccess);
Log.Info("Running DS Client");
bool result = client.Run();
Log.Info("Client run completed. Result=" + result);
t.Interrupt();
// Check maximum number of containers on each NMs
int[] maxRunningContainersOnNMs = mon.GetMaxRunningContainersReport();
// Check no container allocated on NM[0]
NUnit.Framework.Assert.AreEqual(0, maxRunningContainersOnNMs[0]);
// Check there're some containers allocated on NM[1]
NUnit.Framework.Assert.IsTrue(maxRunningContainersOnNMs[1] > 0);
}
/// <exception cref="System.Exception"/>
protected override void ServiceStop()
{
if (stopped.GetAndSet(true))
{
// return if already stopped
return;
}
if (allocatorThread != null)
{
allocatorThread.Interrupt();
try
{
allocatorThread.Join();
}
catch (Exception ie)
{
Log.Warn("InterruptedException while stopping", ie);
}
}
if (isApplicationMasterRegistered && shouldUnregister)
{
Unregister();
}
base.ServiceStop();
}
private void CancelJobCommit()
{
lock (this)
{
Sharpen.Thread threadCommitting = jobCommitThread;
if (threadCommitting != null && threadCommitting.IsAlive())
{
Log.Info("Cancelling commit");
threadCommitting.Interrupt();
// wait up to configured timeout for commit thread to finish
long now = context.GetClock().GetTime();
long timeoutTimestamp = now + commitThreadCancelTimeoutMs;
try
{
while (jobCommitThread == threadCommitting && now > timeoutTimestamp)
{
Sharpen.Runtime.Wait(this, now - timeoutTimestamp);
now = context.GetClock().GetTime();
}
}
catch (Exception)
{
}
}
}
}
public void Run()
{
lock (this)
{
while (!terminated && callingThread.IsAlive())
{
try
{
if (0 < deadline)
{
long delay = deadline - Now();
if (delay <= 0)
{
deadline = 0;
callingThread.Interrupt();
}
else
{
Sharpen.Runtime.Wait(this, delay);
}
}
else
{
Sharpen.Runtime.Wait(this, 1000);
}
}
catch (Exception)
{
}
}
}
}
/// <exception cref="System.Exception"/>
internal virtual void Teardown()
{
if (bkthread != null)
{
bkthread.Interrupt();
bkthread.Join();
}
}
/// <exception cref="NSch.JSchException"></exception>
internal static Socket CreateSocket(string host, int port, int timeout)
{
Socket socket = null;
if (timeout == 0)
{
try
{
socket = Sharpen.Extensions.CreateSocket(host, port);
return(socket);
}
catch (Exception e)
{
string message = e.Message;
if (e is Exception)
{
throw new JSchException(message, (Exception)e);
}
throw new JSchException(message);
}
}
string _host = host;
int _port = port;
Socket[] sockp = new Socket[1];
Exception[] ee = new Exception[1];
string message_1 = string.Empty;
Sharpen.Thread tmp = new Sharpen.Thread(new _Runnable_350(sockp, _host, _port, ee
));
tmp.SetName("Opening Socket " + host);
tmp.Start();
try
{
tmp.Join(timeout);
message_1 = "timeout: ";
}
catch (Exception)
{
}
if (sockp[0] != null && sockp[0].Connected)
{
socket = sockp[0];
}
else
{
message_1 += "socket is not established";
if (ee[0] != null)
{
message_1 = ee[0].ToString();
}
tmp.Interrupt();
tmp = null;
throw new JSchException(message_1);
}
return(socket);
}
/// <exception cref="System.Exception"/>
protected override void ServiceStop()
{
stopped = true;
// this could be called before background thread is established
if (speculationBackgroundThread != null)
{
speculationBackgroundThread.Interrupt();
}
base.ServiceStop();
}
public virtual void Stop()
{
Log.D(Log.TagChangeTracker, "%s: Changed tracker asked to stop", this);
running = false;
thread.Interrupt();
if (request != null)
{
request.Abort();
}
Stopped();
}
/// <summary>Stop this daemon.</summary>
/// <remarks>Stop this daemon.</remarks>
public virtual void Stop()
{
lock (this)
{
if (acceptThread != null)
{
run = false;
acceptThread.Interrupt();
}
}
}
public void Stop()
{
Log.D(Tag, this + ": changed tracker asked to stop");
running = false;
thread.Interrupt();
if (Request != null)
{
tokenSource.Cancel();
}
Stopped();
}
public virtual void Stop()
{
Log.D(Database.Tag, "changed tracker asked to stop");
running = false;
thread.Interrupt();
if (request != null)
{
request.Abort();
}
Stopped();
}
/// <summary>Shut down this instance of the datanode.</summary>
/// <remarks>
/// Shut down this instance of the datanode.
/// Returns only after shutdown is complete.
/// </remarks>
public virtual void Shutdown()
{
shouldRun = false;
if (checkpointThread != null)
{
checkpointThread.Interrupt();
try
{
checkpointThread.Join(10000);
}
catch (Exception)
{
Log.Info("Interrupted waiting to join on checkpointer thread");
Sharpen.Thread.CurrentThread().Interrupt();
}
}
// maintain status
try
{
if (infoServer != null)
{
infoServer.Stop();
infoServer = null;
}
}
catch (Exception e)
{
Log.Warn("Exception shutting down SecondaryNameNode", e);
}
if (nameNodeStatusBeanName != null)
{
MBeans.Unregister(nameNodeStatusBeanName);
nameNodeStatusBeanName = null;
}
try
{
if (checkpointImage != null)
{
checkpointImage.Close();
checkpointImage = null;
}
}
catch (IOException e)
{
Log.Warn("Exception while closing CheckpointStorage", e);
}
if (namesystem != null)
{
namesystem.Shutdown();
namesystem = null;
}
}
/// <exception cref="System.Exception"/>
protected override void ServiceStop()
{
if (stopped.GetAndSet(true))
{
// return if already stopped
return;
}
// shutdown any containers that might be left running
ShutdownAllContainers();
if (eventHandlingThread != null)
{
eventHandlingThread.Interrupt();
}
if (launcherPool != null)
{
launcherPool.ShutdownNow();
}
base.ServiceStop();
}
/// <summary>Interrupt the writing thread and wait until it dies</summary>
/// <exception cref="System.IO.IOException">the waiting is interrupted</exception>
public virtual void StopWriter(long xceiverStopTimeout)
{
if (writer != null && writer != Sharpen.Thread.CurrentThread() && writer.IsAlive(
))
{
writer.Interrupt();
try
{
writer.Join(xceiverStopTimeout);
if (writer.IsAlive())
{
string msg = "Join on writer thread " + writer + " timed out";
DataNode.Log.Warn(msg + "\n" + StringUtils.GetStackTrace(writer));
throw new IOException(msg);
}
}
catch (Exception)
{
throw new IOException("Waiting for writer thread is interrupted.");
}
}
}
/// <exception cref="System.Exception"/>
protected override void ServiceStop()
{
if (drainEventsOnStop)
{
blockNewEvents = true;
Log.Info("AsyncDispatcher is draining to stop, igonring any new events.");
long endTime = Runtime.CurrentTimeMillis() + GetConfig().GetLong(YarnConfiguration
.DispatcherDrainEventsTimeout, YarnConfiguration.DefaultDispatcherDrainEventsTimeout
);
lock (waitForDrained)
{
while (!drained && eventHandlingThread != null && eventHandlingThread.IsAlive() &&
Runtime.CurrentTimeMillis() < endTime)
{
Sharpen.Runtime.Wait(waitForDrained, 1000);
Log.Info("Waiting for AsyncDispatcher to drain. Thread state is :" + eventHandlingThread
.GetState());
}
}
}
stopped = true;
if (eventHandlingThread != null)
{
eventHandlingThread.Interrupt();
try
{
eventHandlingThread.Join();
}
catch (Exception ie)
{
Log.Warn("Interrupted Exception while stopping", ie);
}
}
// stop all the components
base.ServiceStop();
}
/// <exception cref="NSch.JSchException"></exception>
internal static Socket CreateSocket(string host, int port, int timeout)
{
Socket socket = null;
if (timeout == 0)
{
try
{
socket = Sharpen.Extensions.CreateSocket(host, port);
return socket;
}
catch (Exception e)
{
string message = e.ToString();
if (e is Exception)
{
throw new JSchException(message, (Exception)e);
}
throw new JSchException(message);
}
}
string _host = host;
int _port = port;
Socket[] sockp = new Socket[1];
Exception[] ee = new Exception[1];
string message_1 = string.Empty;
Sharpen.Thread tmp = new Sharpen.Thread(new _Runnable_350(sockp, _host, _port, ee
));
tmp.SetName("Opening Socket " + host);
tmp.Start();
try
{
tmp.Join(timeout);
message_1 = "timeout: ";
}
catch (Exception)
{
}
if (sockp[0] != null && sockp[0].Connected)
{
socket = sockp[0];
}
else
{
message_1 += "socket is not established";
if (ee[0] != null)
{
message_1 = ee[0].ToString();
}
tmp.Interrupt();
tmp = null;
throw new JSchException(message_1);
}
return socket;
}
/// <summary>
/// When an InterruptedException is sent to a thread calling
/// FileChannel#read, the FileChannel is immediately closed and the
/// thread gets an exception.
/// </summary>
/// <remarks>
/// When an InterruptedException is sent to a thread calling
/// FileChannel#read, the FileChannel is immediately closed and the
/// thread gets an exception. This effectively means that we might have
/// someone asynchronously calling close() on the file descriptors we use
/// in BlockReaderLocal. So when unreferencing a ShortCircuitReplica in
/// ShortCircuitCache#unref, we should check if the FileChannel objects
/// are still open. If not, we should purge the replica to avoid giving
/// it out to any future readers.
/// This is a regression test for HDFS-6227: Short circuit read failed
/// due to ClosedChannelException.
/// Note that you may still get ClosedChannelException errors if two threads
/// are reading from the same replica and an InterruptedException is delivered
/// to one of them.
/// </remarks>
/// <exception cref="System.Exception"/>
public virtual void TestPurgingClosedReplicas()
{
BlockReaderTestUtil.EnableBlockReaderFactoryTracing();
AtomicInteger replicasCreated = new AtomicInteger(0);
AtomicBoolean testFailed = new AtomicBoolean(false);
DFSInputStream.tcpReadsDisabledForTesting = true;
BlockReaderFactory.createShortCircuitReplicaInfoCallback = new _ShortCircuitReplicaCreator_443
(replicasCreated);
TemporarySocketDirectory sockDir = new TemporarySocketDirectory();
Configuration conf = CreateShortCircuitConf("testPurgingClosedReplicas", sockDir);
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).Build();
cluster.WaitActive();
DistributedFileSystem dfs = cluster.GetFileSystem();
string TestFile = "/test_file";
int TestFileLen = 4095;
int Seed = unchecked ((int)(0xFADE0));
DistributedFileSystem fs = (DistributedFileSystem)FileSystem.Get(cluster.GetURI(0
), conf);
DFSTestUtil.CreateFile(fs, new Path(TestFile), TestFileLen, (short)1, Seed);
Semaphore sem = Sharpen.Extensions.CreateSemaphore(0);
IList <LocatedBlock> locatedBlocks = cluster.GetNameNode().GetRpcServer().GetBlockLocations
(TestFile, 0, TestFileLen).GetLocatedBlocks();
LocatedBlock lblock = locatedBlocks[0];
// first block
byte[] buf = new byte[TestFileLen];
Runnable readerRunnable = new _Runnable_471(cluster, lblock, TestFileLen, sem, buf
, testFailed);
Sharpen.Thread thread = new Sharpen.Thread(readerRunnable);
thread.Start();
// While the thread is reading, send it interrupts.
// These should trigger a ClosedChannelException.
while (thread.IsAlive())
{
sem.AcquireUninterruptibly();
thread.Interrupt();
sem.Release();
}
NUnit.Framework.Assert.IsFalse(testFailed.Get());
// We should be able to read from the file without
// getting a ClosedChannelException.
BlockReader blockReader = null;
try
{
blockReader = BlockReaderTestUtil.GetBlockReader(cluster, lblock, 0, TestFileLen);
blockReader.ReadFully(buf, 0, TestFileLen);
}
finally
{
if (blockReader != null)
{
blockReader.Close();
}
}
byte[] expected = DFSTestUtil.CalculateFileContentsFromSeed(Seed, TestFileLen);
NUnit.Framework.Assert.IsTrue(Arrays.Equals(buf, expected));
// Another ShortCircuitReplica object should have been created.
NUnit.Framework.Assert.AreEqual(2, replicasCreated.Get());
dfs.Close();
cluster.Shutdown();
sockDir.Close();
}
