public virtual void TestReduceOutOfDiskSpace()
{
Log.Info("testReduceOutOfDiskSpace");
Fetcher <Text, Text> underTest = new TestFetcher.FakeFetcher <Text, Text>(job, id,
ss, mm, r, metrics, except, key, connection);
string replyHash = SecureShuffleUtils.GenerateHash(Sharpen.Runtime.GetBytesForString
(encHash), key);
ShuffleHeader header = new ShuffleHeader(map1ID.ToString(), 10, 10, 1);
ByteArrayOutputStream bout = new ByteArrayOutputStream();
header.Write(new DataOutputStream(bout));
ByteArrayInputStream @in = new ByteArrayInputStream(bout.ToByteArray());
Org.Mockito.Mockito.When(connection.GetResponseCode()).ThenReturn(200);
Org.Mockito.Mockito.When(connection.GetHeaderField(ShuffleHeader.HttpHeaderName))
.ThenReturn(ShuffleHeader.DefaultHttpHeaderName);
Org.Mockito.Mockito.When(connection.GetHeaderField(ShuffleHeader.HttpHeaderVersion
)).ThenReturn(ShuffleHeader.DefaultHttpHeaderVersion);
Org.Mockito.Mockito.When(connection.GetHeaderField(SecureShuffleUtils.HttpHeaderReplyUrlHash
)).ThenReturn(replyHash);
Org.Mockito.Mockito.When(connection.GetInputStream()).ThenReturn(@in);
Org.Mockito.Mockito.When(mm.Reserve(Matchers.Any <TaskAttemptID>(), Matchers.AnyLong
(), Matchers.AnyInt())).ThenThrow(new DiskChecker.DiskErrorException("No disk space available"
));
underTest.CopyFromHost(host);
Org.Mockito.Mockito.Verify(ss).ReportLocalError(Matchers.Any <IOException>());
}
public virtual void TestStorageReportHasStorageTypeAndState()
{
// Make sure we are not testing with the default type, that would not
// be a very good test.
NUnit.Framework.Assert.AreNotSame(storageType, StorageType.Default);
NameNode nn = cluster.GetNameNode();
DataNode dn = cluster.GetDataNodes()[0];
// Insert a spy object for the NN RPC.
DatanodeProtocolClientSideTranslatorPB nnSpy = DataNodeTestUtils.SpyOnBposToNN(dn
, nn);
// Trigger a heartbeat so there is an interaction with the spy
// object.
DataNodeTestUtils.TriggerHeartbeat(dn);
// Verify that the callback passed in the expected parameters.
ArgumentCaptor <StorageReport[]> captor = ArgumentCaptor.ForClass <StorageReport[]>
();
Org.Mockito.Mockito.Verify(nnSpy).SendHeartbeat(Matchers.Any <DatanodeRegistration
>(), captor.Capture(), Matchers.AnyLong(), Matchers.AnyLong(), Matchers.AnyInt()
, Matchers.AnyInt(), Matchers.AnyInt(), Org.Mockito.Mockito.Any <VolumeFailureSummary
>());
StorageReport[] reports = captor.GetValue();
foreach (StorageReport report in reports)
{
Assert.AssertThat(report.GetStorage().GetStorageType(), IS.Is(storageType));
Assert.AssertThat(report.GetStorage().GetState(), IS.Is(DatanodeStorage.State.Normal
));
}
}
/// <summary>Ensure that a delayed IBR is generated for a block deleted on the DN.</summary>
/// <exception cref="System.Exception"/>
/// <exception cref="System.IO.IOException"/>
public virtual void TestReportBlockDeleted()
{
try
{
// Trigger a block report to reset the IBR timer.
DataNodeTestUtils.TriggerBlockReport(singletonDn);
// Spy on calls from the DN to the NN
DatanodeProtocolClientSideTranslatorPB nnSpy = SpyOnDnCallsToNn();
InjectBlockDeleted();
// Sleep for a very short time since IBR is generated
// asynchronously.
Sharpen.Thread.Sleep(2000);
// Ensure that no block report was generated immediately.
// Deleted blocks are reported when the IBR timer elapses.
Org.Mockito.Mockito.Verify(nnSpy, Org.Mockito.Mockito.Times(0)).BlockReceivedAndDeleted
(Matchers.Any <DatanodeRegistration>(), Matchers.AnyString(), Matchers.Any <StorageReceivedDeletedBlocks
[]>());
// Trigger a block report, this also triggers an IBR.
DataNodeTestUtils.TriggerBlockReport(singletonDn);
Sharpen.Thread.Sleep(2000);
// Ensure that the deleted block is reported.
Org.Mockito.Mockito.Verify(nnSpy, Org.Mockito.Mockito.Times(1)).BlockReceivedAndDeleted
(Matchers.Any <DatanodeRegistration>(), Matchers.AnyString(), Matchers.Any <StorageReceivedDeletedBlocks
[]>());
}
finally
{
cluster.Shutdown();
cluster = null;
}
}
/// <summary>
/// Determine whether all path matches pass
/// </summary>
/// <param name="stack">List of strings to try to match</param>
/// <returns>Tuple of passing status and matching values</returns>
public (bool, List <string>) MatchesAll(IEnumerable <string> stack)
{
// If no path matches are defined, we fail out
if (Matchers == null || !Matchers.Any())
{
return(false, new List <string>());
}
// Initialize the value list
List <string> values = new List <string>();
// Loop through all path matches and make sure all pass
foreach (var pathMatch in Matchers)
{
(bool match, string value) = pathMatch.Match(stack);
if (!match)
{
return(false, new List <string>());
}
else
{
values.Add(value);
}
}
return(true, values);
}
/// <exception cref="System.Exception"/>
public virtual void TestJobKill()
{
clientDelegate = Org.Mockito.Mockito.Mock <ClientServiceDelegate>();
Org.Mockito.Mockito.When(clientDelegate.GetJobStatus(Matchers.Any <JobID>())).ThenReturn
(new JobStatus(jobId, 0f, 0f, 0f, 0f, JobStatus.State.Prep, JobPriority.High, "tmp"
, "tmp", "tmp", "tmp"));
Org.Mockito.Mockito.When(clientDelegate.KillJob(Matchers.Any <JobID>())).ThenReturn
(true);
Org.Mockito.Mockito.DoAnswer(new _Answer_177(this)).When(clientCache).GetClient(Matchers.Any
<JobID>());
yarnRunner.KillJob(jobId);
Org.Mockito.Mockito.Verify(resourceMgrDelegate).KillApplication(appId);
Org.Mockito.Mockito.When(clientDelegate.GetJobStatus(Matchers.Any <JobID>())).ThenReturn
(new JobStatus(jobId, 0f, 0f, 0f, 0f, JobStatus.State.Running, JobPriority.High,
"tmp", "tmp", "tmp", "tmp"));
yarnRunner.KillJob(jobId);
Org.Mockito.Mockito.Verify(clientDelegate).KillJob(jobId);
Org.Mockito.Mockito.When(clientDelegate.GetJobStatus(Matchers.Any <JobID>())).ThenReturn
(null);
Org.Mockito.Mockito.When(resourceMgrDelegate.GetApplicationReport(Matchers.Any <ApplicationId
>())).ThenReturn(ApplicationReport.NewInstance(appId, null, "tmp", "tmp", "tmp",
"tmp", 0, null, YarnApplicationState.Finished, "tmp", "tmp", 0l, 0l, FinalApplicationStatus
.Succeeded, null, null, 0f, "tmp", null));
yarnRunner.KillJob(jobId);
Org.Mockito.Mockito.Verify(clientDelegate).KillJob(jobId);
}
private static ClientResponse MockEntityClientResponse(TimelineClientImpl client,
ClientResponse.Status status, bool hasError, bool hasRuntimeError)
{
ClientResponse response = Org.Mockito.Mockito.Mock <ClientResponse>();
if (hasRuntimeError)
{
Org.Mockito.Mockito.DoThrow(new ClientHandlerException(new ConnectException())).When
(client).DoPostingObject(Matchers.Any <TimelineEntities>(), Matchers.Any <string>(
));
return(response);
}
Org.Mockito.Mockito.DoReturn(response).When(client).DoPostingObject(Matchers.Any <
TimelineEntities>(), Matchers.Any <string>());
Org.Mockito.Mockito.When(response.GetClientResponseStatus()).ThenReturn(status);
TimelinePutResponse.TimelinePutError error = new TimelinePutResponse.TimelinePutError
();
error.SetEntityId("test entity id");
error.SetEntityType("test entity type");
error.SetErrorCode(TimelinePutResponse.TimelinePutError.IoException);
TimelinePutResponse putResponse = new TimelinePutResponse();
if (hasError)
{
putResponse.AddError(error);
}
Org.Mockito.Mockito.When(response.GetEntity <TimelinePutResponse>()).ThenReturn(putResponse
);
return(response);
}
public virtual void TestContainerLocalizerClosesFilesystems()
{
// mocked generics
// verify filesystems are closed when localizer doesn't fail
FileContext fs = FileContext.GetLocalFSFileContext();
spylfs = Org.Mockito.Mockito.Spy(fs.GetDefaultFileSystem());
ContainerLocalizer localizer = SetupContainerLocalizerForTest();
Org.Mockito.Mockito.DoNothing().When(localizer).LocalizeFiles(Matchers.Any <LocalizationProtocol
>(), Matchers.Any <CompletionService>(), Matchers.Any <UserGroupInformation>());
Org.Mockito.Mockito.Verify(localizer, Org.Mockito.Mockito.Never()).CloseFileSystems
(Matchers.Any <UserGroupInformation>());
localizer.RunLocalization(nmAddr);
Org.Mockito.Mockito.Verify(localizer).CloseFileSystems(Matchers.Any <UserGroupInformation
>());
spylfs = Org.Mockito.Mockito.Spy(fs.GetDefaultFileSystem());
// verify filesystems are closed when localizer fails
localizer = SetupContainerLocalizerForTest();
Org.Mockito.Mockito.DoThrow(new YarnRuntimeException("Forced Failure")).When(localizer
).LocalizeFiles(Matchers.Any <LocalizationProtocol>(), Matchers.Any <CompletionService
>(), Matchers.Any <UserGroupInformation>());
Org.Mockito.Mockito.Verify(localizer, Org.Mockito.Mockito.Never()).CloseFileSystems
(Matchers.Any <UserGroupInformation>());
localizer.RunLocalization(nmAddr);
Org.Mockito.Mockito.Verify(localizer).CloseFileSystems(Matchers.Any <UserGroupInformation
>());
}
/// <summary>Test that a full block report is sent after hot swapping volumes</summary>
/// <exception cref="System.IO.IOException"/>
/// <exception cref="Org.Apache.Hadoop.Conf.ReconfigurationException"/>
public virtual void TestFullBlockReportAfterRemovingVolumes()
{
Configuration conf = new Configuration();
conf.SetLong(DFSConfigKeys.DfsBlockSizeKey, BlockSize);
// Similar to TestTriggerBlockReport, set a really long value for
// dfs.heartbeat.interval, so that incremental block reports and heartbeats
// won't be sent during this test unless they're triggered
// manually.
conf.SetLong(DFSConfigKeys.DfsBlockreportIntervalMsecKey, 10800000L);
conf.SetLong(DFSConfigKeys.DfsHeartbeatIntervalKey, 1080L);
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(2).Build();
cluster.WaitActive();
DataNode dn = cluster.GetDataNodes()[0];
DatanodeProtocolClientSideTranslatorPB spy = DataNodeTestUtils.SpyOnBposToNN(dn,
cluster.GetNameNode());
// Remove a data dir from datanode
FilePath dataDirToKeep = new FilePath(cluster.GetDataDirectory(), "data1");
dn.ReconfigurePropertyImpl(DFSConfigKeys.DfsDatanodeDataDirKey, dataDirToKeep.ToString
());
// We should get 1 full report
Org.Mockito.Mockito.Verify(spy, Org.Mockito.Mockito.Timeout(60000).Times(1)).BlockReport
(Matchers.Any <DatanodeRegistration>(), Matchers.AnyString(), Matchers.Any <StorageBlockReport
[]>(), Matchers.Any <BlockReportContext>());
}
// FIXME:
// Disabled this test because currently, when shutdown hook triggered at
// lastRetry in RM view, cleanup will not do. This should be supported after
// YARN-2261 completed
// @Test (timeout = 30000)
/// <exception cref="System.IO.IOException"/>
public virtual void TestDeletionofStagingOnKillLastTry()
{
conf.Set(MRJobConfig.MapreduceJobDir, stagingJobDir);
fs = Org.Mockito.Mockito.Mock <FileSystem>();
Org.Mockito.Mockito.When(fs.Delete(Matchers.Any <Path>(), Matchers.AnyBoolean())).
ThenReturn(true);
//Staging Dir exists
string user = UserGroupInformation.GetCurrentUser().GetShortUserName();
Path stagingDir = MRApps.GetStagingAreaDir(conf, user);
Org.Mockito.Mockito.When(fs.Exists(stagingDir)).ThenReturn(true);
ApplicationId appId = ApplicationId.NewInstance(Runtime.CurrentTimeMillis(), 0);
ApplicationAttemptId attemptId = ApplicationAttemptId.NewInstance(appId, 1);
JobId jobid = recordFactory.NewRecordInstance <JobId>();
jobid.SetAppId(appId);
ContainerAllocator mockAlloc = Org.Mockito.Mockito.Mock <ContainerAllocator>();
MRAppMaster appMaster = new TestStagingCleanup.TestMRApp(this, attemptId, mockAlloc
);
//no retry
appMaster.Init(conf);
NUnit.Framework.Assert.IsTrue("appMaster.isLastAMRetry() is false", appMaster.IsLastAMRetry
());
//simulate the process being killed
MRAppMaster.MRAppMasterShutdownHook hook = new MRAppMaster.MRAppMasterShutdownHook
(appMaster);
hook.Run();
NUnit.Framework.Assert.IsTrue("MRAppMaster isn't stopped", appMaster.IsInState(Service.STATE
.Stopped));
Org.Mockito.Mockito.Verify(fs).Delete(stagingJobPath, true);
}
/// <exception cref="System.Exception"/>
public virtual void TestCopyFromHostWithRetryUnreserve()
{
InMemoryMapOutput <Text, Text> immo = Org.Mockito.Mockito.Mock <InMemoryMapOutput>(
);
Fetcher <Text, Text> underTest = new TestFetcher.FakeFetcher <Text, Text>(jobWithRetry
, id, ss, mm, r, metrics, except, key, connection);
string replyHash = SecureShuffleUtils.GenerateHash(Sharpen.Runtime.GetBytesForString
(encHash), key);
Org.Mockito.Mockito.When(connection.GetResponseCode()).ThenReturn(200);
Org.Mockito.Mockito.When(connection.GetHeaderField(SecureShuffleUtils.HttpHeaderReplyUrlHash
)).ThenReturn(replyHash);
ShuffleHeader header = new ShuffleHeader(map1ID.ToString(), 10, 10, 1);
ByteArrayOutputStream bout = new ByteArrayOutputStream();
header.Write(new DataOutputStream(bout));
ByteArrayInputStream @in = new ByteArrayInputStream(bout.ToByteArray());
Org.Mockito.Mockito.When(connection.GetInputStream()).ThenReturn(@in);
Org.Mockito.Mockito.When(connection.GetHeaderField(ShuffleHeader.HttpHeaderName))
.ThenReturn(ShuffleHeader.DefaultHttpHeaderName);
Org.Mockito.Mockito.When(connection.GetHeaderField(ShuffleHeader.HttpHeaderVersion
)).ThenReturn(ShuffleHeader.DefaultHttpHeaderVersion);
// Verify that unreserve occurs if an exception happens after shuffle
// buffer is reserved.
Org.Mockito.Mockito.When(mm.Reserve(Matchers.Any <TaskAttemptID>(), Matchers.AnyLong
(), Matchers.AnyInt())).ThenReturn(immo);
Org.Mockito.Mockito.DoThrow(new IOException("forced error")).When(immo).Shuffle(Matchers.Any
<MapHost>(), Matchers.Any <InputStream>(), Matchers.AnyLong(), Matchers.AnyLong()
, Matchers.Any <ShuffleClientMetrics>(), Matchers.Any <Reporter>());
underTest.CopyFromHost(host);
Org.Mockito.Mockito.Verify(immo).Abort();
}
public static Connection GetConnection()
{
Connection connection = Org.Mockito.Mockito.Mock <DriverForTest.FakeConnection>();
try
{
Statement statement = Org.Mockito.Mockito.Mock <Statement>();
ResultSet results = Org.Mockito.Mockito.Mock <ResultSet>();
Org.Mockito.Mockito.When(results.GetLong(1)).ThenReturn(15L);
Org.Mockito.Mockito.When(statement.ExecuteQuery(Matchers.Any <string>())).ThenReturn
(results);
Org.Mockito.Mockito.When(connection.CreateStatement()).ThenReturn(statement);
DatabaseMetaData metadata = Org.Mockito.Mockito.Mock <DatabaseMetaData>();
Org.Mockito.Mockito.When(metadata.GetDatabaseProductName()).ThenReturn("Test");
Org.Mockito.Mockito.When(connection.GetMetaData()).ThenReturn(metadata);
PreparedStatement reparedStatement0 = Org.Mockito.Mockito.Mock <PreparedStatement>
();
Org.Mockito.Mockito.When(connection.PrepareStatement(AnyString())).ThenReturn(reparedStatement0
);
PreparedStatement preparedStatement = Org.Mockito.Mockito.Mock <PreparedStatement>
();
ResultSet resultSet = Org.Mockito.Mockito.Mock <ResultSet>();
Org.Mockito.Mockito.When(resultSet.Next()).ThenReturn(false);
Org.Mockito.Mockito.When(preparedStatement.ExecuteQuery()).ThenReturn(resultSet);
Org.Mockito.Mockito.When(connection.PrepareStatement(AnyString(), AnyInt(), AnyInt
())).ThenReturn(preparedStatement);
}
catch (SQLException)
{
}
return(connection);
}
/// <exception cref="System.Exception"/>
public virtual void TestCopyFromHostCompressFailure()
{
InMemoryMapOutput <Text, Text> immo = Org.Mockito.Mockito.Mock <InMemoryMapOutput>(
);
Fetcher <Text, Text> underTest = new TestFetcher.FakeFetcher <Text, Text>(job, id,
ss, mm, r, metrics, except, key, connection);
string replyHash = SecureShuffleUtils.GenerateHash(Sharpen.Runtime.GetBytesForString
(encHash), key);
Org.Mockito.Mockito.When(connection.GetResponseCode()).ThenReturn(200);
Org.Mockito.Mockito.When(connection.GetHeaderField(SecureShuffleUtils.HttpHeaderReplyUrlHash
)).ThenReturn(replyHash);
ShuffleHeader header = new ShuffleHeader(map1ID.ToString(), 10, 10, 1);
ByteArrayOutputStream bout = new ByteArrayOutputStream();
header.Write(new DataOutputStream(bout));
ByteArrayInputStream @in = new ByteArrayInputStream(bout.ToByteArray());
Org.Mockito.Mockito.When(connection.GetInputStream()).ThenReturn(@in);
Org.Mockito.Mockito.When(connection.GetHeaderField(ShuffleHeader.HttpHeaderName))
.ThenReturn(ShuffleHeader.DefaultHttpHeaderName);
Org.Mockito.Mockito.When(connection.GetHeaderField(ShuffleHeader.HttpHeaderVersion
)).ThenReturn(ShuffleHeader.DefaultHttpHeaderVersion);
Org.Mockito.Mockito.When(mm.Reserve(Matchers.Any <TaskAttemptID>(), Matchers.AnyLong
(), Matchers.AnyInt())).ThenReturn(immo);
Org.Mockito.Mockito.DoThrow(new InternalError()).When(immo).Shuffle(Matchers.Any <
MapHost>(), Matchers.Any <InputStream>(), Matchers.AnyLong(), Matchers.AnyLong(),
Matchers.Any <ShuffleClientMetrics>(), Matchers.Any <Reporter>());
underTest.CopyFromHost(host);
Org.Mockito.Mockito.Verify(connection).AddRequestProperty(SecureShuffleUtils.HttpHeaderUrlHash
, encHash);
Org.Mockito.Mockito.Verify(ss, Org.Mockito.Mockito.Times(1)).CopyFailed(map1ID, host
, true, false);
}
public virtual void TestCopyFromHostIncompatibleShuffleVersionWithRetry()
{
string replyHash = SecureShuffleUtils.GenerateHash(Sharpen.Runtime.GetBytesForString
(encHash), key);
Org.Mockito.Mockito.When(connection.GetResponseCode()).ThenReturn(200);
Org.Mockito.Mockito.When(connection.GetHeaderField(ShuffleHeader.HttpHeaderName))
.ThenReturn("mapreduce").ThenReturn("other").ThenReturn("other");
Org.Mockito.Mockito.When(connection.GetHeaderField(ShuffleHeader.HttpHeaderVersion
)).ThenReturn("1.0.1").ThenReturn("1.0.0").ThenReturn("1.0.1");
Org.Mockito.Mockito.When(connection.GetHeaderField(SecureShuffleUtils.HttpHeaderReplyUrlHash
)).ThenReturn(replyHash);
ByteArrayInputStream @in = new ByteArrayInputStream(new byte[0]);
Org.Mockito.Mockito.When(connection.GetInputStream()).ThenReturn(@in);
for (int i = 0; i < 3; ++i)
{
Fetcher <Text, Text> underTest = new TestFetcher.FakeFetcher <Text, Text>(jobWithRetry
, id, ss, mm, r, metrics, except, key, connection);
underTest.CopyFromHost(host);
}
Org.Mockito.Mockito.Verify(connection, Org.Mockito.Mockito.Times(3)).AddRequestProperty
(SecureShuffleUtils.HttpHeaderUrlHash, encHash);
Org.Mockito.Mockito.Verify(allErrs, Org.Mockito.Mockito.Times(3)).Increment(1);
Org.Mockito.Mockito.Verify(ss, Org.Mockito.Mockito.Times(3)).CopyFailed(map1ID, host
, false, false);
Org.Mockito.Mockito.Verify(ss, Org.Mockito.Mockito.Times(3)).CopyFailed(map2ID, host
, false, false);
Org.Mockito.Mockito.Verify(ss, Org.Mockito.Mockito.Times(3)).PutBackKnownMapOutput
(Matchers.Any <MapHost>(), Matchers.Eq(map1ID));
Org.Mockito.Mockito.Verify(ss, Org.Mockito.Mockito.Times(3)).PutBackKnownMapOutput
(Matchers.Any <MapHost>(), Matchers.Eq(map2ID));
}
public virtual void TestCopyFromHostBogusHeader()
{
Fetcher <Text, Text> underTest = new TestFetcher.FakeFetcher <Text, Text>(job, id,
ss, mm, r, metrics, except, key, connection);
string replyHash = SecureShuffleUtils.GenerateHash(Sharpen.Runtime.GetBytesForString
(encHash), key);
Org.Mockito.Mockito.When(connection.GetResponseCode()).ThenReturn(200);
Org.Mockito.Mockito.When(connection.GetHeaderField(ShuffleHeader.HttpHeaderName))
.ThenReturn(ShuffleHeader.DefaultHttpHeaderName);
Org.Mockito.Mockito.When(connection.GetHeaderField(ShuffleHeader.HttpHeaderVersion
)).ThenReturn(ShuffleHeader.DefaultHttpHeaderVersion);
Org.Mockito.Mockito.When(connection.GetHeaderField(SecureShuffleUtils.HttpHeaderReplyUrlHash
)).ThenReturn(replyHash);
ByteArrayInputStream @in = new ByteArrayInputStream(Sharpen.Runtime.GetBytesForString
("\u00010 BOGUS DATA\nBOGUS DATA\nBOGUS DATA\n"));
Org.Mockito.Mockito.When(connection.GetInputStream()).ThenReturn(@in);
underTest.CopyFromHost(host);
Org.Mockito.Mockito.Verify(connection).AddRequestProperty(SecureShuffleUtils.HttpHeaderUrlHash
, encHash);
Org.Mockito.Mockito.Verify(allErrs).Increment(1);
Org.Mockito.Mockito.Verify(ss).CopyFailed(map1ID, host, true, false);
Org.Mockito.Mockito.Verify(ss).CopyFailed(map2ID, host, true, false);
Org.Mockito.Mockito.Verify(ss).PutBackKnownMapOutput(Matchers.Any <MapHost>(), Matchers.Eq
(map1ID));
Org.Mockito.Mockito.Verify(ss).PutBackKnownMapOutput(Matchers.Any <MapHost>(), Matchers.Eq
(map2ID));
}
public virtual void TestRMDownRestoreForJobStatusBeforeGetAMReport()
{
Configuration conf = new YarnConfiguration();
conf.SetInt(MRJobConfig.MrClientMaxRetries, 3);
conf.Set(MRConfig.FrameworkName, MRConfig.YarnFrameworkName);
conf.SetBoolean(MRJobConfig.JobAmAccessDisabled, !isAMReachableFromClient);
MRClientProtocol historyServerProxy = Org.Mockito.Mockito.Mock <MRClientProtocol>(
);
Org.Mockito.Mockito.When(historyServerProxy.GetJobReport(Matchers.Any <Org.Apache.Hadoop.Mapreduce.V2.Api.Protocolrecords.GetJobReportRequest
>())).ThenReturn(GetJobReportResponse());
ResourceMgrDelegate rmDelegate = Org.Mockito.Mockito.Mock <ResourceMgrDelegate>();
try
{
Org.Mockito.Mockito.When(rmDelegate.GetApplicationReport(jobId.GetAppId())).ThenThrow
(new UndeclaredThrowableException(new IOException("Connection refuced1"))).ThenThrow
(new UndeclaredThrowableException(new IOException("Connection refuced2"))).ThenReturn
(GetFinishedApplicationReport());
ClientServiceDelegate clientServiceDelegate = new ClientServiceDelegate(conf, rmDelegate
, oldJobId, historyServerProxy);
JobStatus jobStatus = clientServiceDelegate.GetJobStatus(oldJobId);
Org.Mockito.Mockito.Verify(rmDelegate, Org.Mockito.Mockito.Times(3)).GetApplicationReport
(Matchers.Any <ApplicationId>());
NUnit.Framework.Assert.IsNotNull(jobStatus);
}
catch (YarnException e)
{
throw new IOException(e);
}
}
public virtual void TestDeletionofStaging()
{
conf.Set(MRJobConfig.MapreduceJobDir, stagingJobDir);
fs = Org.Mockito.Mockito.Mock <FileSystem>();
Org.Mockito.Mockito.When(fs.Delete(Matchers.Any <Path>(), Matchers.AnyBoolean())).
ThenReturn(true);
//Staging Dir exists
string user = UserGroupInformation.GetCurrentUser().GetShortUserName();
Path stagingDir = MRApps.GetStagingAreaDir(conf, user);
Org.Mockito.Mockito.When(fs.Exists(stagingDir)).ThenReturn(true);
ApplicationId appId = ApplicationId.NewInstance(Runtime.CurrentTimeMillis(), 0);
ApplicationAttemptId attemptId = ApplicationAttemptId.NewInstance(appId, 1);
JobId jobid = recordFactory.NewRecordInstance <JobId>();
jobid.SetAppId(appId);
ContainerAllocator mockAlloc = Org.Mockito.Mockito.Mock <ContainerAllocator>();
NUnit.Framework.Assert.IsTrue(MRJobConfig.DefaultMrAmMaxAttempts > 1);
MRAppMaster appMaster = new TestStagingCleanup.TestMRApp(this, attemptId, mockAlloc
, JobStateInternal.Running, MRJobConfig.DefaultMrAmMaxAttempts);
appMaster.Init(conf);
appMaster.Start();
appMaster.ShutDownJob();
//test whether notifyIsLastAMRetry called
NUnit.Framework.Assert.AreEqual(true, ((TestStagingCleanup.TestMRApp)appMaster).GetTestIsLastAMRetry
());
Org.Mockito.Mockito.Verify(fs).Delete(stagingJobPath, true);
}
/// <exception cref="System.IO.IOException"/>
private void TestRMDownForJobStatusBeforeGetAMReport(Configuration conf, int noOfRetries
)
{
conf.Set(MRConfig.FrameworkName, MRConfig.YarnFrameworkName);
conf.SetBoolean(MRJobConfig.JobAmAccessDisabled, !isAMReachableFromClient);
MRClientProtocol historyServerProxy = Org.Mockito.Mockito.Mock <MRClientProtocol>(
);
ResourceMgrDelegate rmDelegate = Org.Mockito.Mockito.Mock <ResourceMgrDelegate>();
try
{
Org.Mockito.Mockito.When(rmDelegate.GetApplicationReport(jobId.GetAppId())).ThenThrow
(new UndeclaredThrowableException(new IOException("Connection refuced1"))).ThenThrow
(new UndeclaredThrowableException(new IOException("Connection refuced2"))).ThenThrow
(new UndeclaredThrowableException(new IOException("Connection refuced3")));
ClientServiceDelegate clientServiceDelegate = new ClientServiceDelegate(conf, rmDelegate
, oldJobId, historyServerProxy);
try
{
clientServiceDelegate.GetJobStatus(oldJobId);
NUnit.Framework.Assert.Fail("It should throw exception after retries");
}
catch (IOException e)
{
System.Console.Out.WriteLine("fail to get job status,and e=" + e.ToString());
}
Org.Mockito.Mockito.Verify(rmDelegate, Org.Mockito.Mockito.Times(noOfRetries)).GetApplicationReport
(Matchers.Any <ApplicationId>());
}
catch (YarnException e)
{
throw new IOException(e);
}
}
/// <exception cref="System.Exception"/>
public virtual void TestRefreshSuperUserGroupsConfiguration()
{
string[] args = new string[] { "-refreshSuperUserGroupsConfiguration" };
NUnit.Framework.Assert.AreEqual(0, rmAdminCLI.Run(args));
Org.Mockito.Mockito.Verify(admin).RefreshSuperUserGroupsConfiguration(Matchers.Any
<RefreshSuperUserGroupsConfigurationRequest>());
}
/// <summary>
/// Determine whether all content matches pass
/// </summary>
/// <param name="fileContent">Byte array representing the file contents</param>
/// <returns>Tuple of passing status and matching positions</returns>
public (bool, List <int>) MatchesAll(byte[] fileContent)
{
// If no content matches are defined, we fail out
if (Matchers == null || !Matchers.Any())
{
return(false, new List <int>());
}
// Initialize the position list
List <int> positions = new List <int>();
// Loop through all content matches and make sure all pass
foreach (var contentMatch in Matchers)
{
(bool match, int position) = contentMatch.Match(fileContent);
if (!match)
{
return(false, new List <int>());
}
else
{
positions.Add(position);
}
}
return(true, positions);
}
/// <exception cref="System.Exception"/>
public virtual void TestRefreshNodes()
{
string[] args = new string[] { "-refreshNodes" };
NUnit.Framework.Assert.AreEqual(0, rmAdminCLI.Run(args));
Org.Mockito.Mockito.Verify(admin).RefreshNodes(Matchers.Any <RefreshNodesRequest>(
));
}
public virtual void TestCommitBlockSynchronization()
{
INodeFile file = MockFileUnderConstruction();
Block block = new Block(blockId, length, genStamp);
FSNamesystem namesystemSpy = MakeNameSystemSpy(block, file);
DatanodeID[] newTargets = new DatanodeID[0];
ExtendedBlock lastBlock = new ExtendedBlock();
namesystemSpy.CommitBlockSynchronization(lastBlock, genStamp, length, false, false
, newTargets, null);
// Repeat the call to make sure it does not throw
namesystemSpy.CommitBlockSynchronization(lastBlock, genStamp, length, false, false
, newTargets, null);
// Simulate 'completing' the block.
BlockInfoContiguous completedBlockInfo = new BlockInfoContiguous(block, (short)1);
completedBlockInfo.SetBlockCollection(file);
completedBlockInfo.SetGenerationStamp(genStamp);
Org.Mockito.Mockito.DoReturn(completedBlockInfo).When(namesystemSpy).GetStoredBlock
(Matchers.Any <Block>());
Org.Mockito.Mockito.DoReturn(completedBlockInfo).When(file).GetLastBlock();
// Repeat the call to make sure it does not throw
namesystemSpy.CommitBlockSynchronization(lastBlock, genStamp, length, false, false
, newTargets, null);
}
public virtual void TestContainerMetricsFlow()
{
string Err = "Error in number of records";
// Create a dummy MetricsSystem
MetricsSystem system = Org.Mockito.Mockito.Mock <MetricsSystem>();
Org.Mockito.Mockito.DoReturn(this).When(system).Register(Matchers.AnyString(), Matchers.AnyString
(), Matchers.Any());
MetricsCollectorImpl collector = new MetricsCollectorImpl();
ContainerId containerId = Org.Mockito.Mockito.Mock <ContainerId>();
ContainerMetrics metrics = ContainerMetrics.ForContainer(containerId, 100, 1);
metrics.RecordMemoryUsage(1024);
metrics.GetMetrics(collector, true);
NUnit.Framework.Assert.AreEqual(Err, 0, collector.GetRecords().Count);
Sharpen.Thread.Sleep(110);
metrics.GetMetrics(collector, true);
NUnit.Framework.Assert.AreEqual(Err, 1, collector.GetRecords().Count);
collector.Clear();
Sharpen.Thread.Sleep(110);
metrics.GetMetrics(collector, true);
NUnit.Framework.Assert.AreEqual(Err, 1, collector.GetRecords().Count);
collector.Clear();
metrics.Finished();
metrics.GetMetrics(collector, true);
NUnit.Framework.Assert.AreEqual(Err, 1, collector.GetRecords().Count);
collector.Clear();
metrics.GetMetrics(collector, true);
NUnit.Framework.Assert.AreEqual(Err, 0, collector.GetRecords().Count);
Sharpen.Thread.Sleep(110);
metrics.GetMetrics(collector, true);
NUnit.Framework.Assert.AreEqual(Err, 0, collector.GetRecords().Count);
}
public virtual void TestEmptyDownload()
{
JobConf conf = new JobConf();
conf.SetClass("fs.mock.impl", typeof(TestLocalDistributedCacheManager.MockFileSystem
), typeof(FileSystem));
URI mockBase = new URI("mock://test-nn1/");
Org.Mockito.Mockito.When(mockfs.GetUri()).ThenReturn(mockBase);
Path working = new Path("mock://test-nn1/user/me/");
Org.Mockito.Mockito.When(mockfs.GetWorkingDirectory()).ThenReturn(working);
Org.Mockito.Mockito.When(mockfs.ResolvePath(Matchers.Any <Path>())).ThenAnswer(new
_Answer_190());
Org.Mockito.Mockito.When(mockfs.GetFileStatus(Matchers.Any <Path>())).ThenAnswer(new
_Answer_197());
Org.Mockito.Mockito.When(mockfs.GetConf()).ThenReturn(conf);
Org.Mockito.Mockito.When(mockfs.Open(Matchers.Any <Path>(), Matchers.AnyInt())).ThenAnswer
(new _Answer_206());
conf.Set(MRJobConfig.CacheFiles, string.Empty);
conf.Set(MRConfig.LocalDir, localDir.GetAbsolutePath());
LocalDistributedCacheManager manager = new LocalDistributedCacheManager();
try
{
manager.Setup(conf);
}
finally
{
manager.Close();
}
}
public virtual void TestContainerMetricsLimit()
{
string Err = "Error in number of records";
MetricsSystem system = Org.Mockito.Mockito.Mock <MetricsSystem>();
Org.Mockito.Mockito.DoReturn(this).When(system).Register(Matchers.AnyString(), Matchers.AnyString
(), Matchers.Any());
MetricsCollectorImpl collector = new MetricsCollectorImpl();
ContainerId containerId = Org.Mockito.Mockito.Mock <ContainerId>();
ContainerMetrics metrics = ContainerMetrics.ForContainer(containerId, 100, 1);
int anyPmemLimit = 1024;
int anyVmemLimit = 2048;
int anyVcores = 10;
string anyProcessId = "1234";
metrics.RecordResourceLimit(anyVmemLimit, anyPmemLimit, anyVcores);
metrics.RecordProcessId(anyProcessId);
Sharpen.Thread.Sleep(110);
metrics.GetMetrics(collector, true);
NUnit.Framework.Assert.AreEqual(Err, 1, collector.GetRecords().Count);
MetricsRecord record = collector.GetRecords()[0];
MetricsRecords.AssertTag(record, ContainerMetrics.ProcessidInfo.Name(), anyProcessId
);
MetricsRecords.AssertMetric(record, ContainerMetrics.PmemLimitMetricName, anyPmemLimit
);
MetricsRecords.AssertMetric(record, ContainerMetrics.VmemLimitMetricName, anyVmemLimit
);
MetricsRecords.AssertMetric(record, ContainerMetrics.VcoreLimitMetricName, anyVcores
);
collector.Clear();
}
/// <exception cref="System.Exception"/>
public virtual void TestJobSubmissionFailure()
{
Org.Mockito.Mockito.When(resourceMgrDelegate.SubmitApplication(Matchers.Any <ApplicationSubmissionContext
>())).ThenReturn(appId);
ApplicationReport report = Org.Mockito.Mockito.Mock <ApplicationReport>();
Org.Mockito.Mockito.When(report.GetApplicationId()).ThenReturn(appId);
Org.Mockito.Mockito.When(report.GetDiagnostics()).ThenReturn(failString);
Org.Mockito.Mockito.When(report.GetYarnApplicationState()).ThenReturn(YarnApplicationState
.Failed);
Org.Mockito.Mockito.When(resourceMgrDelegate.GetApplicationReport(appId)).ThenReturn
(report);
Credentials credentials = new Credentials();
FilePath jobxml = new FilePath(testWorkDir, "job.xml");
OutputStream @out = new FileOutputStream(jobxml);
conf.WriteXml(@out);
@out.Close();
try
{
yarnRunner.SubmitJob(jobId, testWorkDir.GetAbsolutePath().ToString(), credentials
);
}
catch (IOException io)
{
Log.Info("Logging exception:", io);
NUnit.Framework.Assert.IsTrue(io.GetLocalizedMessage().Contains(failString));
}
}
public virtual void TestRetriesOnFailures()
{
ContainerManagementProtocol mockProxy = Org.Mockito.Mockito.Mock <ContainerManagementProtocol
>();
StartContainersResponse mockResponse = Org.Mockito.Mockito.Mock <StartContainersResponse
>();
Org.Mockito.Mockito.When(mockProxy.StartContainers(Matchers.Any <StartContainersRequest
>())).ThenThrow(new NMNotYetReadyException("foo")).ThenReturn(mockResponse);
Configuration conf = new Configuration();
conf.SetInt(YarnConfiguration.RmAmMaxAttempts, 1);
conf.SetInt(YarnConfiguration.ClientNmConnectRetryIntervalMs, 1);
DrainDispatcher dispatcher = new DrainDispatcher();
MockRM rm = new _MockRMWithCustomAMLauncher_206(dispatcher, mockProxy, conf, null
);
rm.Start();
MockNM nm1 = rm.RegisterNode("127.0.0.1:1234", 5120);
RMApp app = rm.SubmitApp(2000);
ApplicationAttemptId appAttemptId = app.GetCurrentAppAttempt().GetAppAttemptId();
// kick the scheduling
nm1.NodeHeartbeat(true);
dispatcher.Await();
rm.WaitForState(appAttemptId, RMAppAttemptState.Launched, 500);
}
/// <summary>
/// Get a mock
/// <see cref="Org.Apache.Hadoop.Yarn.Server.Resourcemanager.RMContext"/>
/// for use in test cases.
/// </summary>
/// <returns>
/// a mock
/// <see cref="Org.Apache.Hadoop.Yarn.Server.Resourcemanager.RMContext"/>
/// for use in test cases
/// </returns>
public static RMContext GetMockRMContext()
{
// Null dispatcher
Dispatcher nullDispatcher = new _Dispatcher_73();
// No op
ContainerAllocationExpirer cae = new ContainerAllocationExpirer(nullDispatcher);
Configuration conf = new Configuration();
RMApplicationHistoryWriter writer = Org.Mockito.Mockito.Mock <RMApplicationHistoryWriter
>();
RMContextImpl rmContext = new RMContextImpl(nullDispatcher, cae, null, null, null
, new AMRMTokenSecretManager(conf, null), new RMContainerTokenSecretManager(conf
), new NMTokenSecretManagerInRM(conf), new ClientToAMTokenSecretManagerInRM());
RMNodeLabelsManager nlm = Org.Mockito.Mockito.Mock <RMNodeLabelsManager>();
Org.Mockito.Mockito.When(nlm.GetQueueResource(Matchers.Any <string>(), Matchers.Any
<Set>(), Matchers.Any <Resource>())).ThenAnswer(new _Answer_105());
Org.Mockito.Mockito.When(nlm.GetResourceByLabel(Matchers.Any <string>(), Matchers.Any
<Resource>())).ThenAnswer(new _Answer_114());
rmContext.SetNodeLabelManager(nlm);
rmContext.SetSystemMetricsPublisher(Org.Mockito.Mockito.Mock <SystemMetricsPublisher
>());
rmContext.SetRMApplicationHistoryWriter(Org.Mockito.Mockito.Mock <RMApplicationHistoryWriter
>());
return(rmContext);
}
public virtual void TestRetriesOnAMConnectionFailures()
{
if (!isAMReachableFromClient)
{
return;
}
ResourceMgrDelegate rm = Org.Mockito.Mockito.Mock <ResourceMgrDelegate>();
Org.Mockito.Mockito.When(rm.GetApplicationReport(TypeConverter.ToYarn(oldJobId).GetAppId
())).ThenReturn(GetRunningApplicationReport("am1", 78));
// throw exception in 1st, 2nd, 3rd and 4th call of getJobReport, and
// succeed in the 5th call.
MRClientProtocol amProxy = Org.Mockito.Mockito.Mock <MRClientProtocol>();
Org.Mockito.Mockito.When(amProxy.GetJobReport(Matchers.Any <Org.Apache.Hadoop.Mapreduce.V2.Api.Protocolrecords.GetJobReportRequest
>())).ThenThrow(new RuntimeException("11")).ThenThrow(new RuntimeException("22")
).ThenThrow(new RuntimeException("33")).ThenThrow(new RuntimeException("44")).ThenReturn
(GetJobReportResponse());
Configuration conf = new YarnConfiguration();
conf.Set(MRConfig.FrameworkName, MRConfig.YarnFrameworkName);
conf.SetBoolean(MRJobConfig.JobAmAccessDisabled, !isAMReachableFromClient);
ClientServiceDelegate clientServiceDelegate = new _ClientServiceDelegate_167(amProxy
, conf, rm, oldJobId, null);
JobStatus jobStatus = clientServiceDelegate.GetJobStatus(oldJobId);
NUnit.Framework.Assert.IsNotNull(jobStatus);
// assert maxClientRetry is not decremented.
NUnit.Framework.Assert.AreEqual(conf.GetInt(MRJobConfig.MrClientMaxRetries, MRJobConfig
.DefaultMrClientMaxRetries), clientServiceDelegate.GetMaxClientRetry());
Org.Mockito.Mockito.Verify(amProxy, Org.Mockito.Mockito.Times(5)).GetJobReport(Matchers.Any
<Org.Apache.Hadoop.Mapreduce.V2.Api.Protocolrecords.GetJobReportRequest>());
}
/// <summary>Add a received block entry and then replace it.</summary>
/// <remarks>
/// Add a received block entry and then replace it. Ensure that a single
/// IBR is generated and that pending receive request state is cleared.
/// This test case verifies the failure in HDFS-5922.
/// </remarks>
/// <exception cref="System.Exception"/>
/// <exception cref="System.IO.IOException"/>
public virtual void TestReplaceReceivedBlock()
{
try
{
// Spy on calls from the DN to the NN
DatanodeProtocolClientSideTranslatorPB nnSpy = SpyOnDnCallsToNn();
InjectBlockReceived();
InjectBlockReceived();
// Overwrite the existing entry.
// Sleep for a very short time since IBR is generated
// asynchronously.
Sharpen.Thread.Sleep(2000);
// Ensure that the received block is reported.
Org.Mockito.Mockito.Verify(nnSpy, Org.Mockito.Mockito.AtLeastOnce()).BlockReceivedAndDeleted
(Matchers.Any <DatanodeRegistration>(), Matchers.AnyString(), Matchers.Any <StorageReceivedDeletedBlocks
[]>());
// Ensure that no more IBRs are pending.
NUnit.Framework.Assert.IsFalse(actor.HasPendingIBR());
}
finally
{
cluster.Shutdown();
cluster = null;
}
}
public override void Start()
{
ahsClient = Org.Mockito.Mockito.Mock <ApplicationHistoryProtocol>();
try
{
Org.Mockito.Mockito.When(ahsClient.GetApplicationReport(Matchers.Any <GetApplicationReportRequest
>())).ThenReturn(mockResponse);
Org.Mockito.Mockito.When(ahsClient.GetApplications(Matchers.Any <GetApplicationsRequest
>())).ThenReturn(mockAppResponse);
Org.Mockito.Mockito.When(ahsClient.GetApplicationAttemptReport(Matchers.Any <GetApplicationAttemptReportRequest
>())).ThenReturn(mockAttemptResponse);
Org.Mockito.Mockito.When(ahsClient.GetApplicationAttempts(Matchers.Any <GetApplicationAttemptsRequest
>())).ThenReturn(mockAppAttemptsResponse);
Org.Mockito.Mockito.When(ahsClient.GetContainers(Matchers.Any <GetContainersRequest
>())).ThenReturn(mockContainersResponse);
Org.Mockito.Mockito.When(ahsClient.GetContainerReport(Matchers.Any <GetContainerReportRequest
>())).ThenReturn(mockContainerResponse);
}
catch (YarnException)
{
NUnit.Framework.Assert.Fail("Exception is not expected.");
}
catch (IOException)
{
NUnit.Framework.Assert.Fail("Exception is not expected.");
}
}