private void Compare(Org.Apache.Hadoop.Security.Token.Token <BlockTokenIdentifier>
expected, Org.Apache.Hadoop.Security.Token.Token <BlockTokenIdentifier> actual)
{
NUnit.Framework.Assert.IsTrue(Arrays.Equals(expected.GetIdentifier(), actual.GetIdentifier
()));
NUnit.Framework.Assert.IsTrue(Arrays.Equals(expected.GetPassword(), actual.GetPassword
()));
NUnit.Framework.Assert.AreEqual(expected.GetKind(), actual.GetKind());
NUnit.Framework.Assert.AreEqual(expected.GetService(), actual.GetService());
}
public virtual void TestClusterGetDelegationToken()
{
Configuration conf = new Configuration(false);
Cluster cluster = null;
try
{
conf = new Configuration();
conf.Set(MRConfig.FrameworkName, MRConfig.YarnFrameworkName);
cluster = new Cluster(conf);
YARNRunner yrunner = (YARNRunner)cluster.GetClient();
GetDelegationTokenResponse getDTResponse = recordFactory.NewRecordInstance <GetDelegationTokenResponse
>();
Token rmDTToken = recordFactory.NewRecordInstance <Token>();
rmDTToken.SetIdentifier(ByteBuffer.Wrap(new byte[2]));
rmDTToken.SetKind("Testclusterkind");
rmDTToken.SetPassword(ByteBuffer.Wrap(Sharpen.Runtime.GetBytesForString("testcluster"
)));
rmDTToken.SetService("0.0.0.0:8032");
getDTResponse.SetRMDelegationToken(rmDTToken);
ApplicationClientProtocol cRMProtocol = Org.Mockito.Mockito.Mock <ApplicationClientProtocol
>();
Org.Mockito.Mockito.When(cRMProtocol.GetDelegationToken(Matchers.Any <GetDelegationTokenRequest
>())).ThenReturn(getDTResponse);
ResourceMgrDelegate rmgrDelegate = new _ResourceMgrDelegate_112(cRMProtocol, new
YarnConfiguration(conf));
yrunner.SetResourceMgrDelegate(rmgrDelegate);
Org.Apache.Hadoop.Security.Token.Token t = cluster.GetDelegationToken(new Text(" "
));
NUnit.Framework.Assert.IsTrue("Token kind is instead " + t.GetKind().ToString(),
"Testclusterkind".Equals(t.GetKind().ToString()));
}
finally
{
if (cluster != null)
{
cluster.Close();
}
}
}
/// <exception cref="System.IO.IOException"/>
private void RecordJobShuffleInfo(JobID jobId, string user, Org.Apache.Hadoop.Security.Token.Token
<JobTokenIdentifier> jobToken)
{
if (stateDb != null)
{
SecurityProtos.TokenProto tokenProto = ((SecurityProtos.TokenProto)SecurityProtos.TokenProto
.NewBuilder().SetIdentifier(ByteString.CopyFrom(jobToken.GetIdentifier())).SetPassword
(ByteString.CopyFrom(jobToken.GetPassword())).SetKind(jobToken.GetKind().ToString
()).SetService(jobToken.GetService().ToString()).Build());
ShuffleHandlerRecoveryProtos.JobShuffleInfoProto proto = ((ShuffleHandlerRecoveryProtos.JobShuffleInfoProto
)ShuffleHandlerRecoveryProtos.JobShuffleInfoProto.NewBuilder().SetUser(user).SetJobToken
(tokenProto).Build());
try
{
stateDb.Put(JniDBFactory.Bytes(jobId.ToString()), proto.ToByteArray());
}
catch (DBException e)
{
throw new IOException("Error storing " + jobId, e);
}
}
AddJobToken(jobId, user, jobToken);
}
/// <exception cref="System.IO.IOException"/>
public virtual GetDelegationTokenResponse GetDelegationToken(GetDelegationTokenRequest
request)
{
UserGroupInformation ugi = UserGroupInformation.GetCurrentUser();
// Verify that the connection is kerberos authenticated
if (!this.IsAllowedDelegationTokenOp())
{
throw new IOException("Delegation Token can be issued only with kerberos authentication"
);
}
GetDelegationTokenResponse response = this.recordFactory.NewRecordInstance <GetDelegationTokenResponse
>();
string user = ugi.GetUserName();
Text owner = new Text(user);
Text realUser = null;
if (ugi.GetRealUser() != null)
{
realUser = new Text(ugi.GetRealUser().GetUserName());
}
MRDelegationTokenIdentifier tokenIdentifier = new MRDelegationTokenIdentifier(owner
, new Text(request.GetRenewer()), realUser);
Org.Apache.Hadoop.Security.Token.Token <MRDelegationTokenIdentifier> realJHSToken =
new Org.Apache.Hadoop.Security.Token.Token <MRDelegationTokenIdentifier>(tokenIdentifier
, this._enclosing.jhsDTSecretManager);
Org.Apache.Hadoop.Yarn.Api.Records.Token mrDToken = Org.Apache.Hadoop.Yarn.Api.Records.Token
.NewInstance(realJHSToken.GetIdentifier(), realJHSToken.GetKind().ToString(), realJHSToken
.GetPassword(), realJHSToken.GetService().ToString());
response.SetDelegationToken(mrDToken);
return(response);
}
/// <exception cref="System.Exception"/>
public LocatedBlocks Answer(InvocationOnMock arg0)
{
LocatedBlocks locatedBlocks = (LocatedBlocks)arg0.CallRealMethod();
foreach (LocatedBlock lb in locatedBlocks.GetLocatedBlocks())
{
Org.Apache.Hadoop.Security.Token.Token <BlockTokenIdentifier> token = lb.GetBlockToken
();
BlockTokenIdentifier id = lb.GetBlockToken().DecodeIdentifier();
id.SetExpiryDate(Time.Now() + 10);
Org.Apache.Hadoop.Security.Token.Token <BlockTokenIdentifier> newToken = new Org.Apache.Hadoop.Security.Token.Token
<BlockTokenIdentifier>(id.GetBytes(), token.GetPassword(), token.GetKind(), token
.GetService());
lb.SetBlockToken(newToken);
}
return(locatedBlocks);
}
internal static bool CheckEqual(Org.Apache.Hadoop.Security.Token.Token <TokenIdentifier
> a, Org.Apache.Hadoop.Security.Token.Token <TokenIdentifier> b)
{
return(Arrays.Equals(a.GetIdentifier(), b.GetIdentifier()) && Arrays.Equals(a.GetPassword
(), b.GetPassword()) && IsEqual(a.GetKind(), b.GetKind()) && IsEqual(a.GetService
(), b.GetService()));
}
/// <summary>test JobConf</summary>
/// <exception cref="System.Exception"/>
public virtual void TestNetworkedJob()
{
// mock creation
MiniMRClientCluster mr = null;
FileSystem fileSys = null;
try
{
mr = CreateMiniClusterWithCapacityScheduler();
JobConf job = new JobConf(mr.GetConfig());
fileSys = FileSystem.Get(job);
fileSys.Delete(testDir, true);
FSDataOutputStream @out = fileSys.Create(inFile, true);
@out.WriteBytes("This is a test file");
@out.Close();
FileInputFormat.SetInputPaths(job, inFile);
FileOutputFormat.SetOutputPath(job, outDir);
job.SetInputFormat(typeof(TextInputFormat));
job.SetOutputFormat(typeof(TextOutputFormat));
job.SetMapperClass(typeof(IdentityMapper));
job.SetReducerClass(typeof(IdentityReducer));
job.SetNumReduceTasks(0);
JobClient client = new JobClient(mr.GetConfig());
RunningJob rj = client.SubmitJob(job);
JobID jobId = rj.GetID();
JobClient.NetworkedJob runningJob = (JobClient.NetworkedJob)client.GetJob(jobId);
runningJob.SetJobPriority(JobPriority.High.ToString());
// test getters
NUnit.Framework.Assert.IsTrue(runningJob.GetConfiguration().ToString().EndsWith("0001/job.xml"
));
NUnit.Framework.Assert.AreEqual(runningJob.GetID(), jobId);
NUnit.Framework.Assert.AreEqual(runningJob.GetJobID(), jobId.ToString());
NUnit.Framework.Assert.AreEqual(runningJob.GetJobName(), "N/A");
NUnit.Framework.Assert.IsTrue(runningJob.GetJobFile().EndsWith(".staging/" + runningJob
.GetJobID() + "/job.xml"));
NUnit.Framework.Assert.IsTrue(runningJob.GetTrackingURL().Length > 0);
NUnit.Framework.Assert.IsTrue(runningJob.MapProgress() == 0.0f);
NUnit.Framework.Assert.IsTrue(runningJob.ReduceProgress() == 0.0f);
NUnit.Framework.Assert.IsTrue(runningJob.CleanupProgress() == 0.0f);
NUnit.Framework.Assert.IsTrue(runningJob.SetupProgress() == 0.0f);
TaskCompletionEvent[] tce = runningJob.GetTaskCompletionEvents(0);
NUnit.Framework.Assert.AreEqual(tce.Length, 0);
NUnit.Framework.Assert.AreEqual(runningJob.GetHistoryUrl(), string.Empty);
NUnit.Framework.Assert.IsFalse(runningJob.IsRetired());
NUnit.Framework.Assert.AreEqual(runningJob.GetFailureInfo(), string.Empty);
NUnit.Framework.Assert.AreEqual(runningJob.GetJobStatus().GetJobName(), "N/A");
NUnit.Framework.Assert.AreEqual(client.GetMapTaskReports(jobId).Length, 0);
try
{
client.GetSetupTaskReports(jobId);
}
catch (YarnRuntimeException e)
{
NUnit.Framework.Assert.AreEqual(e.Message, "Unrecognized task type: JOB_SETUP");
}
try
{
client.GetCleanupTaskReports(jobId);
}
catch (YarnRuntimeException e)
{
NUnit.Framework.Assert.AreEqual(e.Message, "Unrecognized task type: JOB_CLEANUP");
}
NUnit.Framework.Assert.AreEqual(client.GetReduceTaskReports(jobId).Length, 0);
// test ClusterStatus
ClusterStatus status = client.GetClusterStatus(true);
NUnit.Framework.Assert.AreEqual(status.GetActiveTrackerNames().Count, 2);
// it method does not implemented and always return empty array or null;
NUnit.Framework.Assert.AreEqual(status.GetBlacklistedTrackers(), 0);
NUnit.Framework.Assert.AreEqual(status.GetBlacklistedTrackerNames().Count, 0);
NUnit.Framework.Assert.AreEqual(status.GetBlackListedTrackersInfo().Count, 0);
NUnit.Framework.Assert.AreEqual(status.GetJobTrackerStatus(), Cluster.JobTrackerStatus
.Running);
NUnit.Framework.Assert.AreEqual(status.GetMapTasks(), 1);
NUnit.Framework.Assert.AreEqual(status.GetMaxMapTasks(), 20);
NUnit.Framework.Assert.AreEqual(status.GetMaxReduceTasks(), 4);
NUnit.Framework.Assert.AreEqual(status.GetNumExcludedNodes(), 0);
NUnit.Framework.Assert.AreEqual(status.GetReduceTasks(), 1);
NUnit.Framework.Assert.AreEqual(status.GetTaskTrackers(), 2);
NUnit.Framework.Assert.AreEqual(status.GetTTExpiryInterval(), 0);
NUnit.Framework.Assert.AreEqual(status.GetJobTrackerStatus(), Cluster.JobTrackerStatus
.Running);
NUnit.Framework.Assert.AreEqual(status.GetGraylistedTrackers(), 0);
// test read and write
ByteArrayOutputStream dataOut = new ByteArrayOutputStream();
status.Write(new DataOutputStream(dataOut));
ClusterStatus status2 = new ClusterStatus();
status2.ReadFields(new DataInputStream(new ByteArrayInputStream(dataOut.ToByteArray
())));
NUnit.Framework.Assert.AreEqual(status.GetActiveTrackerNames(), status2.GetActiveTrackerNames
());
NUnit.Framework.Assert.AreEqual(status.GetBlackListedTrackersInfo(), status2.GetBlackListedTrackersInfo
());
NUnit.Framework.Assert.AreEqual(status.GetMapTasks(), status2.GetMapTasks());
try
{
}
catch (RuntimeException e)
{
NUnit.Framework.Assert.IsTrue(e.Message.EndsWith("not found on CLASSPATH"));
}
// test taskStatusfilter
JobClient.SetTaskOutputFilter(job, JobClient.TaskStatusFilter.All);
NUnit.Framework.Assert.AreEqual(JobClient.GetTaskOutputFilter(job), JobClient.TaskStatusFilter
.All);
// runningJob.setJobPriority(JobPriority.HIGH.name());
// test default map
NUnit.Framework.Assert.AreEqual(client.GetDefaultMaps(), 20);
NUnit.Framework.Assert.AreEqual(client.GetDefaultReduces(), 4);
NUnit.Framework.Assert.AreEqual(client.GetSystemDir().GetName(), "jobSubmitDir");
// test queue information
JobQueueInfo[] rootQueueInfo = client.GetRootQueues();
NUnit.Framework.Assert.AreEqual(rootQueueInfo.Length, 1);
NUnit.Framework.Assert.AreEqual(rootQueueInfo[0].GetQueueName(), "default");
JobQueueInfo[] qinfo = client.GetQueues();
NUnit.Framework.Assert.AreEqual(qinfo.Length, 1);
NUnit.Framework.Assert.AreEqual(qinfo[0].GetQueueName(), "default");
NUnit.Framework.Assert.AreEqual(client.GetChildQueues("default").Length, 0);
NUnit.Framework.Assert.AreEqual(client.GetJobsFromQueue("default").Length, 1);
NUnit.Framework.Assert.IsTrue(client.GetJobsFromQueue("default")[0].GetJobFile().
EndsWith("/job.xml"));
JobQueueInfo qi = client.GetQueueInfo("default");
NUnit.Framework.Assert.AreEqual(qi.GetQueueName(), "default");
NUnit.Framework.Assert.AreEqual(qi.GetQueueState(), "running");
QueueAclsInfo[] aai = client.GetQueueAclsForCurrentUser();
NUnit.Framework.Assert.AreEqual(aai.Length, 2);
NUnit.Framework.Assert.AreEqual(aai[0].GetQueueName(), "root");
NUnit.Framework.Assert.AreEqual(aai[1].GetQueueName(), "default");
// test token
Org.Apache.Hadoop.Security.Token.Token <DelegationTokenIdentifier> token = client.
GetDelegationToken(new Text(UserGroupInformation.GetCurrentUser().GetShortUserName
()));
NUnit.Framework.Assert.AreEqual(token.GetKind().ToString(), "RM_DELEGATION_TOKEN"
);
// test JobClient
// The following asserts read JobStatus twice and ensure the returned
// JobStatus objects correspond to the same Job.
NUnit.Framework.Assert.AreEqual("Expected matching JobIDs", jobId, ((JobID)client
.GetJob(jobId).GetJobStatus().GetJobID()));
NUnit.Framework.Assert.AreEqual("Expected matching startTimes", rj.GetJobStatus()
.GetStartTime(), client.GetJob(jobId).GetJobStatus().GetStartTime());
}
finally
{
if (fileSys != null)
{
fileSys.Delete(testDir, true);
}
if (mr != null)
{
mr.Stop();
}
}
}