public virtual void TestDeletionofStaging()
{
conf.Set(MRJobConfig.MapreduceJobDir, stagingJobDir);
fs = Org.Mockito.Mockito.Mock <FileSystem>();
Org.Mockito.Mockito.When(fs.Delete(Matchers.Any <Path>(), Matchers.AnyBoolean())).
ThenReturn(true);
//Staging Dir exists
string user = UserGroupInformation.GetCurrentUser().GetShortUserName();
Path stagingDir = MRApps.GetStagingAreaDir(conf, user);
Org.Mockito.Mockito.When(fs.Exists(stagingDir)).ThenReturn(true);
ApplicationId appId = ApplicationId.NewInstance(Runtime.CurrentTimeMillis(), 0);
ApplicationAttemptId attemptId = ApplicationAttemptId.NewInstance(appId, 1);
JobId jobid = recordFactory.NewRecordInstance <JobId>();
jobid.SetAppId(appId);
ContainerAllocator mockAlloc = Org.Mockito.Mockito.Mock <ContainerAllocator>();
NUnit.Framework.Assert.IsTrue(MRJobConfig.DefaultMrAmMaxAttempts > 1);
MRAppMaster appMaster = new TestStagingCleanup.TestMRApp(this, attemptId, mockAlloc
, JobStateInternal.Running, MRJobConfig.DefaultMrAmMaxAttempts);
appMaster.Init(conf);
appMaster.Start();
appMaster.ShutDownJob();
//test whether notifyIsLastAMRetry called
NUnit.Framework.Assert.AreEqual(true, ((TestStagingCleanup.TestMRApp)appMaster).GetTestIsLastAMRetry
());
Org.Mockito.Mockito.Verify(fs).Delete(stagingJobPath, true);
}
// FIXME:
// Disabled this test because currently, when shutdown hook triggered at
// lastRetry in RM view, cleanup will not do. This should be supported after
// YARN-2261 completed
// @Test (timeout = 30000)
/// <exception cref="System.IO.IOException"/>
public virtual void TestDeletionofStagingOnKillLastTry()
{
conf.Set(MRJobConfig.MapreduceJobDir, stagingJobDir);
fs = Org.Mockito.Mockito.Mock <FileSystem>();
Org.Mockito.Mockito.When(fs.Delete(Matchers.Any <Path>(), Matchers.AnyBoolean())).
ThenReturn(true);
//Staging Dir exists
string user = UserGroupInformation.GetCurrentUser().GetShortUserName();
Path stagingDir = MRApps.GetStagingAreaDir(conf, user);
Org.Mockito.Mockito.When(fs.Exists(stagingDir)).ThenReturn(true);
ApplicationId appId = ApplicationId.NewInstance(Runtime.CurrentTimeMillis(), 0);
ApplicationAttemptId attemptId = ApplicationAttemptId.NewInstance(appId, 1);
JobId jobid = recordFactory.NewRecordInstance <JobId>();
jobid.SetAppId(appId);
ContainerAllocator mockAlloc = Org.Mockito.Mockito.Mock <ContainerAllocator>();
MRAppMaster appMaster = new TestStagingCleanup.TestMRApp(this, attemptId, mockAlloc
);
//no retry
appMaster.Init(conf);
NUnit.Framework.Assert.IsTrue("appMaster.isLastAMRetry() is false", appMaster.IsLastAMRetry
());
//simulate the process being killed
MRAppMaster.MRAppMasterShutdownHook hook = new MRAppMaster.MRAppMasterShutdownHook
(appMaster);
hook.Run();
NUnit.Framework.Assert.IsTrue("MRAppMaster isn't stopped", appMaster.IsInState(Service.STATE
.Stopped));
Org.Mockito.Mockito.Verify(fs).Delete(stagingJobPath, true);
}
public virtual void TestProcessEvictableResource()
{
FileSystem fs = Org.Mockito.Mockito.Mock <FileSystem>();
CleanerMetrics metrics = Org.Mockito.Mockito.Mock <CleanerMetrics>();
SCMStore store = Org.Mockito.Mockito.Mock <SCMStore>();
CleanerTask task = CreateSpiedTask(fs, store, metrics, new ReentrantLock());
// mock an evictable resource
Org.Mockito.Mockito.When(store.IsResourceEvictable(Matchers.IsA <string>(), Matchers.IsA
<FileStatus>())).ThenReturn(true);
FileStatus status = Org.Mockito.Mockito.Mock <FileStatus>();
Org.Mockito.Mockito.When(status.GetPath()).ThenReturn(new Path(Root + "/a/b/c/abc"
));
Org.Mockito.Mockito.When(store.RemoveResource(Matchers.IsA <string>())).ThenReturn
(true);
// rename succeeds
Org.Mockito.Mockito.When(fs.Rename(Matchers.IsA <Path>(), Matchers.IsA <Path>())).ThenReturn
(true);
// delete returns true
Org.Mockito.Mockito.When(fs.Delete(Matchers.IsA <Path>(), Matchers.AnyBoolean())).
ThenReturn(true);
// process the resource
task.ProcessSingleResource(status);
// the directory should be renamed
Org.Mockito.Mockito.Verify(fs).Rename(Matchers.Eq(status.GetPath()), Matchers.IsA
<Path>());
// metrics should record a deleted file
Org.Mockito.Mockito.Verify(metrics).ReportAFileDelete();
Org.Mockito.Mockito.Verify(metrics, Org.Mockito.Mockito.Never()).ReportAFileProcess
();
}
/// <exception cref="System.IO.IOException"/>
public virtual void TestNormalizeNodeLabelExpression()
{
// mock queue and scheduler
YarnScheduler scheduler = Org.Mockito.Mockito.Mock <YarnScheduler>();
ICollection <string> queueAccessibleNodeLabels = Sets.NewHashSet();
QueueInfo queueInfo = Org.Mockito.Mockito.Mock <QueueInfo>();
Org.Mockito.Mockito.When(queueInfo.GetQueueName()).ThenReturn("queue");
Org.Mockito.Mockito.When(queueInfo.GetAccessibleNodeLabels()).ThenReturn(queueAccessibleNodeLabels
);
Org.Mockito.Mockito.When(queueInfo.GetDefaultNodeLabelExpression()).ThenReturn(" x "
);
Org.Mockito.Mockito.When(scheduler.GetQueueInfo(Matchers.Any <string>(), Matchers.AnyBoolean
(), Matchers.AnyBoolean())).ThenReturn(queueInfo);
Org.Apache.Hadoop.Yarn.Api.Records.Resource maxResource = Resources.CreateResource
(YarnConfiguration.DefaultRmSchedulerMaximumAllocationMb, YarnConfiguration.DefaultRmSchedulerMaximumAllocationVcores
);
// queue has labels, success cases
try
{
// set queue accessible node labels to [x, y]
queueAccessibleNodeLabels.Clear();
Sharpen.Collections.AddAll(queueAccessibleNodeLabels, Arrays.AsList("x", "y"));
rmContext.GetNodeLabelManager().AddToCluserNodeLabels(ImmutableSet.Of("x", "y"));
Org.Apache.Hadoop.Yarn.Api.Records.Resource resource = Resources.CreateResource(0
, YarnConfiguration.DefaultRmSchedulerMinimumAllocationVcores);
ResourceRequest resReq = BuilderUtils.NewResourceRequest(Org.Mockito.Mockito.Mock
<Priority>(), ResourceRequest.Any, resource, 1);
SchedulerUtils.NormalizeAndvalidateRequest(resReq, maxResource, "queue", scheduler
, rmContext);
NUnit.Framework.Assert.IsTrue(resReq.GetNodeLabelExpression().Equals("x"));
resReq.SetNodeLabelExpression(" y ");
SchedulerUtils.NormalizeAndvalidateRequest(resReq, maxResource, "queue", scheduler
, rmContext);
NUnit.Framework.Assert.IsTrue(resReq.GetNodeLabelExpression().Equals("y"));
}
catch (InvalidResourceRequestException e)
{
Sharpen.Runtime.PrintStackTrace(e);
NUnit.Framework.Assert.Fail("Should be valid when request labels is a subset of queue labels"
);
}
finally
{
rmContext.GetNodeLabelManager().RemoveFromClusterNodeLabels(Arrays.AsList("x", "y"
));
}
}
/// <exception cref="System.Exception"/>
private ContainerLocalizer SetupContainerLocalizerForTest()
{
// mocked generics
// don't actually create dirs
Org.Mockito.Mockito.DoNothing().When(spylfs).Mkdir(Matchers.IsA <Path>(), Matchers.IsA
<FsPermission>(), Matchers.AnyBoolean());
Configuration conf = new Configuration();
FileContext lfs = FileContext.GetFileContext(spylfs, conf);
localDirs = new AList <Path>();
for (int i = 0; i < 4; ++i)
{
localDirs.AddItem(lfs.MakeQualified(new Path(basedir, i + string.Empty)));
}
RecordFactory mockRF = GetMockLocalizerRecordFactory();
ContainerLocalizer concreteLoc = new ContainerLocalizer(lfs, appUser, appId, containerId
, localDirs, mockRF);
ContainerLocalizer localizer = Org.Mockito.Mockito.Spy(concreteLoc);
// return credential stream instead of opening local file
random = new Random();
long seed = random.NextLong();
System.Console.Out.WriteLine("SEED: " + seed);
random.SetSeed(seed);
DataInputBuffer appTokens = CreateFakeCredentials(random, 10);
tokenPath = lfs.MakeQualified(new Path(string.Format(ContainerLocalizer.TokenFileNameFmt
, containerId)));
Org.Mockito.Mockito.DoReturn(new FSDataInputStream(new FakeFSDataInputStream(appTokens
))).When(spylfs).Open(tokenPath);
nmProxy = Org.Mockito.Mockito.Mock <LocalizationProtocol>();
Org.Mockito.Mockito.DoReturn(nmProxy).When(localizer).GetProxy(nmAddr);
Org.Mockito.Mockito.DoNothing().When(localizer).Sleep(Matchers.AnyInt());
// return result instantly for deterministic test
ExecutorService syncExec = Org.Mockito.Mockito.Mock <ExecutorService>();
CompletionService <Path> cs = Org.Mockito.Mockito.Mock <CompletionService>();
Org.Mockito.Mockito.When(cs.Submit(Matchers.IsA <Callable>())).ThenAnswer(new _Answer_279
());
Org.Mockito.Mockito.DoReturn(syncExec).When(localizer).CreateDownloadThreadPool();
Org.Mockito.Mockito.DoReturn(cs).When(localizer).CreateCompletionService(syncExec
);
return(localizer);
}
public virtual void TestNoReplicaUnderRecovery()
{
if (Log.IsDebugEnabled())
{
Log.Debug("Running " + GenericTestUtils.GetMethodName());
}
dn.data.CreateRbw(StorageType.Default, block, false);
try
{
dn.SyncBlock(rBlock, InitBlockRecords(dn));
NUnit.Framework.Assert.Fail("Sync should fail");
}
catch (IOException e)
{
e.Message.StartsWith("Cannot recover ");
}
DatanodeProtocol namenode = dn.GetActiveNamenodeForBP(PoolId);
Org.Mockito.Mockito.Verify(namenode, Org.Mockito.Mockito.Never()).CommitBlockSynchronization
(Matchers.Any <ExtendedBlock>(), Matchers.AnyLong(), Matchers.AnyLong(), Matchers.AnyBoolean
(), Matchers.AnyBoolean(), Matchers.Any <DatanodeID[]>(), Matchers.Any <string[]>(
));
}
public virtual void TestNotMatchedReplicaID()
{
if (Log.IsDebugEnabled())
{
Log.Debug("Running " + GenericTestUtils.GetMethodName());
}
ReplicaInPipelineInterface replicaInfo = dn.data.CreateRbw(StorageType.Default, block
, false).GetReplica();
ReplicaOutputStreams streams = null;
try
{
streams = replicaInfo.CreateStreams(true, DataChecksum.NewDataChecksum(DataChecksum.Type
.Crc32, 512));
streams.GetChecksumOut().Write('a');
dn.data.InitReplicaRecovery(new BlockRecoveryCommand.RecoveringBlock(block, null,
RecoveryId + 1));
try
{
dn.SyncBlock(rBlock, InitBlockRecords(dn));
NUnit.Framework.Assert.Fail("Sync should fail");
}
catch (IOException e)
{
e.Message.StartsWith("Cannot recover ");
}
DatanodeProtocol namenode = dn.GetActiveNamenodeForBP(PoolId);
Org.Mockito.Mockito.Verify(namenode, Org.Mockito.Mockito.Never()).CommitBlockSynchronization
(Matchers.Any <ExtendedBlock>(), Matchers.AnyLong(), Matchers.AnyLong(), Matchers.AnyBoolean
(), Matchers.AnyBoolean(), Matchers.Any <DatanodeID[]>(), Matchers.Any <string[]>(
));
}
finally
{
streams.Close();
}
}
/// <exception cref="System.IO.IOException"/>
private void TestDeletionofStagingOnUnregistrationFailure(int maxAttempts, bool shouldHaveDeleted
)
{
conf.Set(MRJobConfig.MapreduceJobDir, stagingJobDir);
fs = Org.Mockito.Mockito.Mock <FileSystem>();
Org.Mockito.Mockito.When(fs.Delete(Matchers.Any <Path>(), Matchers.AnyBoolean())).
ThenReturn(true);
//Staging Dir exists
string user = UserGroupInformation.GetCurrentUser().GetShortUserName();
Path stagingDir = MRApps.GetStagingAreaDir(conf, user);
Org.Mockito.Mockito.When(fs.Exists(stagingDir)).ThenReturn(true);
ApplicationId appId = ApplicationId.NewInstance(0, 1);
ApplicationAttemptId attemptId = ApplicationAttemptId.NewInstance(appId, 1);
JobId jobid = recordFactory.NewRecordInstance <JobId>();
jobid.SetAppId(appId);
TestStagingCleanup.TestMRApp appMaster = new TestStagingCleanup.TestMRApp(this, attemptId
, null, JobStateInternal.Running, maxAttempts);
appMaster.crushUnregistration = true;
appMaster.Init(conf);
appMaster.Start();
appMaster.ShutDownJob();
((MRAppMaster.RunningAppContext)appMaster.GetContext()).ResetIsLastAMRetry();
if (shouldHaveDeleted)
{
NUnit.Framework.Assert.AreEqual(true, appMaster.IsLastAMRetry());
Org.Mockito.Mockito.Verify(fs).Delete(stagingJobPath, true);
}
else
{
NUnit.Framework.Assert.AreEqual(false, appMaster.IsLastAMRetry());
Org.Mockito.Mockito.Verify(fs, Org.Mockito.Mockito.Never()).Delete(stagingJobPath
, true);
}
}
public virtual void TestNotifySCMFail()
{
Configuration conf = new Configuration();
conf.SetBoolean(YarnConfiguration.SharedCacheEnabled, true);
LocalResource resource = Org.Mockito.Mockito.Mock <LocalResource>();
Path localPath = Org.Mockito.Mockito.Mock <Path>();
Org.Mockito.Mockito.When(localPath.GetName()).ThenReturn("foo.jar");
string user = "******";
FileSystem fs = Org.Mockito.Mockito.Mock <FileSystem>();
// return false when rename is called
Org.Mockito.Mockito.When(fs.Rename(Matchers.IsA <Path>(), Matchers.IsA <Path>())).ThenReturn
(true);
FileSystem localFs = FileSystem.GetLocal(conf);
SharedCacheUploader spied = CreateSpiedUploader(resource, localPath, user, conf,
null, fs, localFs);
// stub verifyAccess() to return true
Org.Mockito.Mockito.DoReturn(true).When(spied).VerifyAccess();
// stub getActualPath()
Org.Mockito.Mockito.DoReturn(localPath).When(spied).GetActualPath();
// stub computeChecksum()
Org.Mockito.Mockito.DoReturn("abcdef0123456789").When(spied).ComputeChecksum(Matchers.IsA
<Path>());
// stub uploadFile() to return true
Org.Mockito.Mockito.DoReturn(true).When(spied).UploadFile(Matchers.IsA <Path>(), Matchers.IsA
<Path>());
// stub notifySharedCacheManager to return true
Org.Mockito.Mockito.DoReturn(false).When(spied).NotifySharedCacheManager(Matchers.IsA
<string>(), Matchers.IsA <string>());
NUnit.Framework.Assert.IsFalse(spied.Call());
Org.Mockito.Mockito.Verify(fs).Delete(Matchers.IsA <Path>(), Matchers.AnyBoolean()
);
}
public virtual void TestReadURL()
{
HttpURLConnection conn = Org.Mockito.Mockito.Mock <HttpURLConnection>();
Org.Mockito.Mockito.DoReturn(new ByteArrayInputStream(FakeLogData)).When(conn).GetInputStream
();
Org.Mockito.Mockito.DoReturn(HttpURLConnection.HttpOk).When(conn).GetResponseCode
();
Org.Mockito.Mockito.DoReturn(Sharpen.Extensions.ToString(FakeLogData.Length)).When
(conn).GetHeaderField("Content-Length");
URLConnectionFactory factory = Org.Mockito.Mockito.Mock <URLConnectionFactory>();
Org.Mockito.Mockito.DoReturn(conn).When(factory).OpenConnection(Org.Mockito.Mockito
.Any <Uri>(), Matchers.AnyBoolean());
Uri url = new Uri("http://localhost/fakeLog");
EditLogInputStream elis = EditLogFileInputStream.FromUrl(factory, url, HdfsConstants
.InvalidTxid, HdfsConstants.InvalidTxid, false);
// Read the edit log and verify that we got all of the data.
EnumMap <FSEditLogOpCodes, Holder <int> > counts = FSImageTestUtil.CountEditLogOpTypes
(elis);
Assert.AssertThat(counts[FSEditLogOpCodes.OpAdd].held, CoreMatchers.Is(1));
Assert.AssertThat(counts[FSEditLogOpCodes.OpSetGenstampV1].held, CoreMatchers.Is(
1));
Assert.AssertThat(counts[FSEditLogOpCodes.OpClose].held, CoreMatchers.Is(1));
// Check that length header was picked up.
NUnit.Framework.Assert.AreEqual(FakeLogData.Length, elis.Length());
elis.Close();
}
/// <exception cref="System.Exception"/>
public virtual void TestCopyFromHostWithRetry()
{
InMemoryMapOutput <Text, Text> immo = Org.Mockito.Mockito.Mock <InMemoryMapOutput>(
);
ss = Org.Mockito.Mockito.Mock <ShuffleSchedulerImpl>();
Fetcher <Text, Text> underTest = new TestFetcher.FakeFetcher <Text, Text>(jobWithRetry
, id, ss, mm, r, metrics, except, key, connection, true);
string replyHash = SecureShuffleUtils.GenerateHash(Sharpen.Runtime.GetBytesForString
(encHash), key);
Org.Mockito.Mockito.When(connection.GetResponseCode()).ThenReturn(200);
Org.Mockito.Mockito.When(connection.GetHeaderField(SecureShuffleUtils.HttpHeaderReplyUrlHash
)).ThenReturn(replyHash);
ShuffleHeader header = new ShuffleHeader(map1ID.ToString(), 10, 10, 1);
ByteArrayOutputStream bout = new ByteArrayOutputStream();
header.Write(new DataOutputStream(bout));
ByteArrayInputStream @in = new ByteArrayInputStream(bout.ToByteArray());
Org.Mockito.Mockito.When(connection.GetInputStream()).ThenReturn(@in);
Org.Mockito.Mockito.When(connection.GetHeaderField(ShuffleHeader.HttpHeaderName))
.ThenReturn(ShuffleHeader.DefaultHttpHeaderName);
Org.Mockito.Mockito.When(connection.GetHeaderField(ShuffleHeader.HttpHeaderVersion
)).ThenReturn(ShuffleHeader.DefaultHttpHeaderVersion);
Org.Mockito.Mockito.When(mm.Reserve(Matchers.Any <TaskAttemptID>(), Matchers.AnyLong
(), Matchers.AnyInt())).ThenReturn(immo);
long retryTime = Time.MonotonicNow();
Org.Mockito.Mockito.DoAnswer(new _Answer_375(retryTime)).When(immo).Shuffle(Matchers.Any
<MapHost>(), Matchers.Any <InputStream>(), Matchers.AnyLong(), Matchers.AnyLong()
, Matchers.Any <ShuffleClientMetrics>(), Matchers.Any <Reporter>());
// Emulate host down for 3 seconds.
underTest.CopyFromHost(host);
Org.Mockito.Mockito.Verify(ss, Org.Mockito.Mockito.Never()).CopyFailed(Matchers.Any
<TaskAttemptID>(), Matchers.Any <MapHost>(), Matchers.AnyBoolean(), Matchers.AnyBoolean
());
}
public virtual void TestFactory()
{
string[] groups = new string[] { "supergroup" };
UserGroupInformation[] ugi = new UserGroupInformation[3];
for (int i = 0; i < ugi.Length; i++)
{
ugi[i] = UserGroupInformation.CreateUserForTesting("user" + i, groups);
}
Org.Mockito.Mockito.DoReturn(new HdfsFileStatus(0, false, 1, 1024, 0, 0, new FsPermission
((short)777), "owner", "group", new byte[0], new byte[0], 1010, 0, null, unchecked (
(byte)0))).When(mcp).GetFileInfo(Matchers.AnyString());
Org.Mockito.Mockito.DoReturn(new HdfsFileStatus(0, false, 1, 1024, 0, 0, new FsPermission
((short)777), "owner", "group", new byte[0], new byte[0], 1010, 0, null, unchecked (
(byte)0))).When(mcp).Create(Matchers.AnyString(), (FsPermission)Matchers.AnyObject
(), Matchers.AnyString(), (EnumSetWritable <CreateFlag>)Matchers.AnyObject(), Matchers.AnyBoolean
(), Matchers.AnyShort(), Matchers.AnyLong(), (CryptoProtocolVersion[])Matchers.AnyObject
());
Configuration conf = new Configuration();
DFSClient c1 = CreateDFSClientAs(ugi[0], conf);
FSDataOutputStream out1 = CreateFsOut(c1, "/out1");
DFSClient c2 = CreateDFSClientAs(ugi[0], conf);
FSDataOutputStream out2 = CreateFsOut(c2, "/out2");
NUnit.Framework.Assert.AreEqual(c1.GetLeaseRenewer(), c2.GetLeaseRenewer());
DFSClient c3 = CreateDFSClientAs(ugi[1], conf);
FSDataOutputStream out3 = CreateFsOut(c3, "/out3");
NUnit.Framework.Assert.IsTrue(c1.GetLeaseRenewer() != c3.GetLeaseRenewer());
DFSClient c4 = CreateDFSClientAs(ugi[1], conf);
FSDataOutputStream out4 = CreateFsOut(c4, "/out4");
NUnit.Framework.Assert.AreEqual(c3.GetLeaseRenewer(), c4.GetLeaseRenewer());
DFSClient c5 = CreateDFSClientAs(ugi[2], conf);
FSDataOutputStream out5 = CreateFsOut(c5, "/out5");
NUnit.Framework.Assert.IsTrue(c1.GetLeaseRenewer() != c5.GetLeaseRenewer());
NUnit.Framework.Assert.IsTrue(c3.GetLeaseRenewer() != c5.GetLeaseRenewer());
}
public virtual void TestByteRange()
{
ByteRangeInputStream.URLOpener oMock = GetMockURLOpener(new Uri("http://test"));
ByteRangeInputStream.URLOpener rMock = GetMockURLOpener(null);
ByteRangeInputStream bris = new TestByteRangeInputStream.ByteRangeInputStreamImpl
(this, oMock, rMock);
bris.Seek(0);
NUnit.Framework.Assert.AreEqual("getPos wrong", 0, bris.GetPos());
bris.Read();
NUnit.Framework.Assert.AreEqual("Initial call made incorrectly (offset check)", 0
, bris.startPos);
NUnit.Framework.Assert.AreEqual("getPos should return 1 after reading one byte",
1, bris.GetPos());
Org.Mockito.Mockito.Verify(oMock, Org.Mockito.Mockito.Times(1)).Connect(0, false);
bris.Read();
NUnit.Framework.Assert.AreEqual("getPos should return 2 after reading two bytes",
2, bris.GetPos());
// No additional connections should have been made (no seek)
Org.Mockito.Mockito.Verify(oMock, Org.Mockito.Mockito.Times(1)).Connect(0, false);
rMock.SetURL(new Uri("http://resolvedurl/"));
bris.Seek(100);
bris.Read();
NUnit.Framework.Assert.AreEqual("Seek to 100 bytes made incorrectly (offset Check)"
, 100, bris.startPos);
NUnit.Framework.Assert.AreEqual("getPos should return 101 after reading one byte"
, 101, bris.GetPos());
Org.Mockito.Mockito.Verify(rMock, Org.Mockito.Mockito.Times(1)).Connect(100, true
);
bris.Seek(101);
bris.Read();
// Seek to 101 should not result in another request
Org.Mockito.Mockito.Verify(rMock, Org.Mockito.Mockito.Times(1)).Connect(100, true
);
Org.Mockito.Mockito.Verify(rMock, Org.Mockito.Mockito.Times(0)).Connect(101, true
);
bris.Seek(2500);
bris.Read();
NUnit.Framework.Assert.AreEqual("Seek to 2500 bytes made incorrectly (offset Check)"
, 2500, bris.startPos);
Org.Mockito.Mockito.DoReturn(GetMockConnection(null)).When(rMock).Connect(Matchers.AnyLong
(), Matchers.AnyBoolean());
bris.Seek(500);
try
{
bris.Read();
NUnit.Framework.Assert.Fail("Exception should be thrown when content-length is not given"
);
}
catch (IOException e)
{
NUnit.Framework.Assert.IsTrue("Incorrect response message: " + e.Message, e.Message
.StartsWith(HttpHeaders.ContentLength + " is missing: "));
}
bris.Close();
}
/// <exception cref="System.IO.IOException"/>
private ByteRangeInputStream.URLOpener GetMockURLOpener(Uri url)
{
ByteRangeInputStream.URLOpener opener = Org.Mockito.Mockito.Mock <ByteRangeInputStream.URLOpener
>(Org.Mockito.Mockito.CallsRealMethods);
opener.SetURL(url);
Org.Mockito.Mockito.DoReturn(GetMockConnection("65535")).When(opener).Connect(Matchers.AnyLong
(), Matchers.AnyBoolean());
return(opener);
}
/// <exception cref="System.IO.IOException"/>
private TestFailureToReadEdits.LimitedEditLogAnswer CauseFailureOnEditLogRead()
{
FSEditLog spyEditLog = NameNodeAdapter.SpyOnEditLog(nn1);
TestFailureToReadEdits.LimitedEditLogAnswer answer = new TestFailureToReadEdits.LimitedEditLogAnswer
();
Org.Mockito.Mockito.DoAnswer(answer).When(spyEditLog).SelectInputStreams(Matchers.AnyLong
(), Matchers.AnyLong(), (MetaRecoveryContext)Matchers.AnyObject(), Matchers.AnyBoolean
());
return(answer);
}
/// <exception cref="System.IO.IOException"/>
public virtual void TestValidateResourceRequestWithErrorLabelsPermission()
{
// mock queue and scheduler
YarnScheduler scheduler = Org.Mockito.Mockito.Mock <YarnScheduler>();
ICollection <string> queueAccessibleNodeLabels = Sets.NewHashSet();
QueueInfo queueInfo = Org.Mockito.Mockito.Mock <QueueInfo>();
Org.Mockito.Mockito.When(queueInfo.GetQueueName()).ThenReturn("queue");
Org.Mockito.Mockito.When(queueInfo.GetAccessibleNodeLabels()).ThenReturn(queueAccessibleNodeLabels
);
Org.Mockito.Mockito.When(scheduler.GetQueueInfo(Matchers.Any <string>(), Matchers.AnyBoolean
(), Matchers.AnyBoolean())).ThenReturn(queueInfo);
Org.Apache.Hadoop.Yarn.Api.Records.Resource maxResource = Resources.CreateResource
(YarnConfiguration.DefaultRmSchedulerMaximumAllocationMb, YarnConfiguration.DefaultRmSchedulerMaximumAllocationVcores
);
// queue has labels, success cases
try
{
// set queue accessible node labesl to [x, y]
queueAccessibleNodeLabels.Clear();
Sharpen.Collections.AddAll(queueAccessibleNodeLabels, Arrays.AsList("x", "y"));
rmContext.GetNodeLabelManager().AddToCluserNodeLabels(ImmutableSet.Of("x", "y"));
Org.Apache.Hadoop.Yarn.Api.Records.Resource resource = Resources.CreateResource(0
, YarnConfiguration.DefaultRmSchedulerMinimumAllocationVcores);
ResourceRequest resReq = BuilderUtils.NewResourceRequest(Org.Mockito.Mockito.Mock
<Priority>(), ResourceRequest.Any, resource, 1);
resReq.SetNodeLabelExpression("x");
SchedulerUtils.NormalizeAndvalidateRequest(resReq, maxResource, "queue", scheduler
, rmContext);
resReq.SetNodeLabelExpression("y");
SchedulerUtils.NormalizeAndvalidateRequest(resReq, maxResource, "queue", scheduler
, rmContext);
resReq.SetNodeLabelExpression(string.Empty);
SchedulerUtils.NormalizeAndvalidateRequest(resReq, maxResource, "queue", scheduler
, rmContext);
resReq.SetNodeLabelExpression(" ");
SchedulerUtils.NormalizeAndvalidateRequest(resReq, maxResource, "queue", scheduler
, rmContext);
}
catch (InvalidResourceRequestException e)
{
Sharpen.Runtime.PrintStackTrace(e);
NUnit.Framework.Assert.Fail("Should be valid when request labels is a subset of queue labels"
);
}
finally
{
rmContext.GetNodeLabelManager().RemoveFromClusterNodeLabels(Arrays.AsList("x", "y"
));
}
// same as above, but cluster node labels don't contains label being
// requested. should fail
try
{
// set queue accessible node labesl to [x, y]
queueAccessibleNodeLabels.Clear();
Sharpen.Collections.AddAll(queueAccessibleNodeLabels, Arrays.AsList("x", "y"));
Org.Apache.Hadoop.Yarn.Api.Records.Resource resource = Resources.CreateResource(0
, YarnConfiguration.DefaultRmSchedulerMinimumAllocationVcores);
ResourceRequest resReq = BuilderUtils.NewResourceRequest(Org.Mockito.Mockito.Mock
<Priority>(), ResourceRequest.Any, resource, 1);
resReq.SetNodeLabelExpression("x");
SchedulerUtils.NormalizeAndvalidateRequest(resReq, maxResource, "queue", scheduler
, rmContext);
NUnit.Framework.Assert.Fail("Should fail");
}
catch (InvalidResourceRequestException)
{
}
// queue has labels, failed cases (when ask a label not included by queue)
try
{
// set queue accessible node labesl to [x, y]
queueAccessibleNodeLabels.Clear();
Sharpen.Collections.AddAll(queueAccessibleNodeLabels, Arrays.AsList("x", "y"));
rmContext.GetNodeLabelManager().AddToCluserNodeLabels(ImmutableSet.Of("x", "y"));
Org.Apache.Hadoop.Yarn.Api.Records.Resource resource = Resources.CreateResource(0
, YarnConfiguration.DefaultRmSchedulerMinimumAllocationVcores);
ResourceRequest resReq = BuilderUtils.NewResourceRequest(Org.Mockito.Mockito.Mock
<Priority>(), ResourceRequest.Any, resource, 1);
resReq.SetNodeLabelExpression("z");
SchedulerUtils.NormalizeAndvalidateRequest(resReq, maxResource, "queue", scheduler
, rmContext);
NUnit.Framework.Assert.Fail("Should fail");
}
catch (InvalidResourceRequestException)
{
}
finally
{
rmContext.GetNodeLabelManager().RemoveFromClusterNodeLabels(Arrays.AsList("x", "y"
));
}
// we don't allow specify more than two node labels in a single expression
// now
try
{
// set queue accessible node labesl to [x, y]
queueAccessibleNodeLabels.Clear();
Sharpen.Collections.AddAll(queueAccessibleNodeLabels, Arrays.AsList("x", "y"));
rmContext.GetNodeLabelManager().AddToCluserNodeLabels(ImmutableSet.Of("x", "y"));
Org.Apache.Hadoop.Yarn.Api.Records.Resource resource = Resources.CreateResource(0
, YarnConfiguration.DefaultRmSchedulerMinimumAllocationVcores);
ResourceRequest resReq = BuilderUtils.NewResourceRequest(Org.Mockito.Mockito.Mock
<Priority>(), ResourceRequest.Any, resource, 1);
resReq.SetNodeLabelExpression("x && y");
SchedulerUtils.NormalizeAndvalidateRequest(resReq, maxResource, "queue", scheduler
, rmContext);
NUnit.Framework.Assert.Fail("Should fail");
}
catch (InvalidResourceRequestException)
{
}
finally
{
rmContext.GetNodeLabelManager().RemoveFromClusterNodeLabels(Arrays.AsList("x", "y"
));
}
// queue doesn't have label, succeed (when request no label)
queueAccessibleNodeLabels.Clear();
try
{
// set queue accessible node labels to empty
queueAccessibleNodeLabels.Clear();
Org.Apache.Hadoop.Yarn.Api.Records.Resource resource = Resources.CreateResource(0
, YarnConfiguration.DefaultRmSchedulerMinimumAllocationVcores);
ResourceRequest resReq = BuilderUtils.NewResourceRequest(Org.Mockito.Mockito.Mock
<Priority>(), ResourceRequest.Any, resource, 1);
SchedulerUtils.NormalizeAndvalidateRequest(resReq, maxResource, "queue", scheduler
, rmContext);
resReq.SetNodeLabelExpression(string.Empty);
SchedulerUtils.NormalizeAndvalidateRequest(resReq, maxResource, "queue", scheduler
, rmContext);
resReq.SetNodeLabelExpression(" ");
SchedulerUtils.NormalizeAndvalidateRequest(resReq, maxResource, "queue", scheduler
, rmContext);
}
catch (InvalidResourceRequestException e)
{
Sharpen.Runtime.PrintStackTrace(e);
NUnit.Framework.Assert.Fail("Should be valid when request labels is empty");
}
// queue doesn't have label, failed (when request any label)
try
{
// set queue accessible node labels to empty
queueAccessibleNodeLabels.Clear();
rmContext.GetNodeLabelManager().AddToCluserNodeLabels(ImmutableSet.Of("x"));
Org.Apache.Hadoop.Yarn.Api.Records.Resource resource = Resources.CreateResource(0
, YarnConfiguration.DefaultRmSchedulerMinimumAllocationVcores);
ResourceRequest resReq = BuilderUtils.NewResourceRequest(Org.Mockito.Mockito.Mock
<Priority>(), ResourceRequest.Any, resource, 1);
resReq.SetNodeLabelExpression("x");
SchedulerUtils.NormalizeAndvalidateRequest(resReq, maxResource, "queue", scheduler
, rmContext);
NUnit.Framework.Assert.Fail("Should fail");
}
catch (InvalidResourceRequestException)
{
}
finally
{
rmContext.GetNodeLabelManager().RemoveFromClusterNodeLabels(Arrays.AsList("x"));
}
// queue is "*", always succeeded
try
{
// set queue accessible node labels to empty
queueAccessibleNodeLabels.Clear();
queueAccessibleNodeLabels.AddItem(RMNodeLabelsManager.Any);
rmContext.GetNodeLabelManager().AddToCluserNodeLabels(ImmutableSet.Of("x", "y", "z"
));
Org.Apache.Hadoop.Yarn.Api.Records.Resource resource = Resources.CreateResource(0
, YarnConfiguration.DefaultRmSchedulerMinimumAllocationVcores);
ResourceRequest resReq = BuilderUtils.NewResourceRequest(Org.Mockito.Mockito.Mock
<Priority>(), ResourceRequest.Any, resource, 1);
resReq.SetNodeLabelExpression("x");
SchedulerUtils.NormalizeAndvalidateRequest(resReq, maxResource, "queue", scheduler
, rmContext);
resReq.SetNodeLabelExpression("y");
SchedulerUtils.NormalizeAndvalidateRequest(resReq, maxResource, "queue", scheduler
, rmContext);
resReq.SetNodeLabelExpression("z");
SchedulerUtils.NormalizeAndvalidateRequest(resReq, maxResource, "queue", scheduler
, rmContext);
}
catch (InvalidResourceRequestException e)
{
Sharpen.Runtime.PrintStackTrace(e);
NUnit.Framework.Assert.Fail("Should be valid when queue can access any labels");
}
finally
{
rmContext.GetNodeLabelManager().RemoveFromClusterNodeLabels(Arrays.AsList("x", "y"
, "z"));
}
// same as above, but cluster node labels don't contains label, should fail
try
{
// set queue accessible node labels to empty
queueAccessibleNodeLabels.Clear();
queueAccessibleNodeLabels.AddItem(RMNodeLabelsManager.Any);
Org.Apache.Hadoop.Yarn.Api.Records.Resource resource = Resources.CreateResource(0
, YarnConfiguration.DefaultRmSchedulerMinimumAllocationVcores);
ResourceRequest resReq = BuilderUtils.NewResourceRequest(Org.Mockito.Mockito.Mock
<Priority>(), ResourceRequest.Any, resource, 1);
resReq.SetNodeLabelExpression("x");
SchedulerUtils.NormalizeAndvalidateRequest(resReq, maxResource, "queue", scheduler
, rmContext);
NUnit.Framework.Assert.Fail("Should fail");
}
catch (InvalidResourceRequestException)
{
}
// we don't allow resource name other than ANY and specify label
try
{
// set queue accessible node labesl to [x, y]
queueAccessibleNodeLabels.Clear();
Sharpen.Collections.AddAll(queueAccessibleNodeLabels, Arrays.AsList("x", "y"));
rmContext.GetNodeLabelManager().AddToCluserNodeLabels(ImmutableSet.Of("x", "y"));
Org.Apache.Hadoop.Yarn.Api.Records.Resource resource = Resources.CreateResource(0
, YarnConfiguration.DefaultRmSchedulerMinimumAllocationVcores);
ResourceRequest resReq = BuilderUtils.NewResourceRequest(Org.Mockito.Mockito.Mock
<Priority>(), "rack", resource, 1);
resReq.SetNodeLabelExpression("x");
SchedulerUtils.NormalizeAndvalidateRequest(resReq, maxResource, "queue", scheduler
, rmContext);
NUnit.Framework.Assert.Fail("Should fail");
}
catch (InvalidResourceRequestException)
{
}
finally
{
rmContext.GetNodeLabelManager().RemoveFromClusterNodeLabels(Arrays.AsList("x", "y"
));
}
// we don't allow resource name other than ANY and specify label even if
// queue has accessible label = *
try
{
// set queue accessible node labesl to *
queueAccessibleNodeLabels.Clear();
Sharpen.Collections.AddAll(queueAccessibleNodeLabels, Arrays.AsList(CommonNodeLabelsManager
.Any));
rmContext.GetNodeLabelManager().AddToCluserNodeLabels(ImmutableSet.Of("x"));
Org.Apache.Hadoop.Yarn.Api.Records.Resource resource = Resources.CreateResource(0
, YarnConfiguration.DefaultRmSchedulerMinimumAllocationVcores);
ResourceRequest resReq = BuilderUtils.NewResourceRequest(Org.Mockito.Mockito.Mock
<Priority>(), "rack", resource, 1);
resReq.SetNodeLabelExpression("x");
SchedulerUtils.NormalizeAndvalidateRequest(resReq, maxResource, "queue", scheduler
, rmContext);
NUnit.Framework.Assert.Fail("Should fail");
}
catch (InvalidResourceRequestException)
{
}
finally
{
rmContext.GetNodeLabelManager().RemoveFromClusterNodeLabels(Arrays.AsList("x"));
}
}
public virtual void TestTokenCompatibilityFor203()
{
Configuration conf = new Configuration();
HftpFileSystem fs = new HftpFileSystem();
Org.Apache.Hadoop.Security.Token.Token <object> token = new Org.Apache.Hadoop.Security.Token.Token
<TokenIdentifier>(new byte[0], new byte[0], DelegationTokenIdentifier.HdfsDelegationKind
, new Text("127.0.0.1:8020"));
Credentials cred = new Credentials();
cred.AddToken(HftpFileSystem.TokenKind, token);
ByteArrayOutputStream os = new ByteArrayOutputStream();
cred.Write(new DataOutputStream(os));
HttpURLConnection conn = Org.Mockito.Mockito.Mock <HttpURLConnection>();
Org.Mockito.Mockito.DoReturn(new ByteArrayInputStream(os.ToByteArray())).When(conn
).GetInputStream();
Org.Mockito.Mockito.DoReturn(HttpURLConnection.HttpOk).When(conn).GetResponseCode
();
URLConnectionFactory factory = Org.Mockito.Mockito.Mock <URLConnectionFactory>();
Org.Mockito.Mockito.DoReturn(conn).When(factory).OpenConnection(Org.Mockito.Mockito
.Any <Uri>(), Matchers.AnyBoolean());
URI uri = new URI("hftp://127.0.0.1:8020");
fs.Initialize(uri, conf);
fs.connectionFactory = factory;
UserGroupInformation ugi = UserGroupInformation.CreateUserForTesting("foo", new string
[] { "bar" });
TokenAspect <HftpFileSystem> tokenAspect = new TokenAspect <HftpFileSystem>(fs, SecurityUtil
.BuildTokenService(uri), HftpFileSystem.TokenKind);
tokenAspect.InitDelegationToken(ugi);
tokenAspect.EnsureTokenInitialized();
NUnit.Framework.Assert.AreSame(HftpFileSystem.TokenKind, fs.GetRenewToken().GetKind
());
Org.Apache.Hadoop.Security.Token.Token <object> tok = (Org.Apache.Hadoop.Security.Token.Token
<object>)Whitebox.GetInternalState(fs, "delegationToken");
NUnit.Framework.Assert.AreNotSame("Not making a copy of the remote token", token,
tok);
NUnit.Framework.Assert.AreEqual(token.GetKind(), tok.GetKind());
}
public virtual void TestAppsRace()
{
// mock up an RM that returns app reports for apps that don't exist
// in the RMApps list
ApplicationId appId = ApplicationId.NewInstance(1, 1);
ApplicationReport mockReport = Org.Mockito.Mockito.Mock <ApplicationReport>();
Org.Mockito.Mockito.When(mockReport.GetApplicationId()).ThenReturn(appId);
GetApplicationsResponse mockAppsResponse = Org.Mockito.Mockito.Mock <GetApplicationsResponse
>();
Org.Mockito.Mockito.When(mockAppsResponse.GetApplicationList()).ThenReturn(Arrays
.AsList(new ApplicationReport[] { mockReport }));
ClientRMService mockClientSvc = Org.Mockito.Mockito.Mock <ClientRMService>();
Org.Mockito.Mockito.When(mockClientSvc.GetApplications(Matchers.IsA <GetApplicationsRequest
>(), Matchers.AnyBoolean())).ThenReturn(mockAppsResponse);
ResourceManager mockRM = Org.Mockito.Mockito.Mock <ResourceManager>();
RMContextImpl rmContext = new RMContextImpl(null, null, null, null, null, null, null
, null, null, null);
Org.Mockito.Mockito.When(mockRM.GetRMContext()).ThenReturn(rmContext);
Org.Mockito.Mockito.When(mockRM.GetClientRMService()).ThenReturn(mockClientSvc);
RMWebServices webSvc = new RMWebServices(mockRM, new Configuration(), Org.Mockito.Mockito.Mock
<HttpServletResponse>());
ICollection <string> emptySet = Collections.UnmodifiableSet(Collections.EmptySet <string
>());
// verify we don't get any apps when querying
HttpServletRequest mockHsr = Org.Mockito.Mockito.Mock <HttpServletRequest>();
AppsInfo appsInfo = webSvc.GetApps(mockHsr, null, emptySet, null, null, null, null
, null, null, null, null, emptySet, emptySet);
NUnit.Framework.Assert.IsTrue(appsInfo.GetApps().IsEmpty());
// verify we don't get an NPE when specifying a final status query
appsInfo = webSvc.GetApps(mockHsr, null, emptySet, "FAILED", null, null, null, null
, null, null, null, emptySet, emptySet);
NUnit.Framework.Assert.IsTrue(appsInfo.GetApps().IsEmpty());
}