public MasterKeyData(int serialNo, SecretKey secretKey)
{
// Underlying secret-key also stored to avoid repetitive encoding and
// decoding the masterKeyRecord bytes.
this.masterKeyRecord = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <MasterKey>();
this.masterKeyRecord.SetKeyId(serialNo);
this.generatedSecretKey = secretKey;
this.masterKeyRecord.SetBytes(ByteBuffer.Wrap(generatedSecretKey.GetEncoded()));
}
/// <summary>Internal method for submitting jobs to the system.</summary>
/// <remarks>
/// Internal method for submitting jobs to the system.
/// <p>The job submission process involves:
/// <ol>
/// <li>
/// Checking the input and output specifications of the job.
/// </li>
/// <li>
/// Computing the
/// <see cref="InputSplit"/>
/// s for the job.
/// </li>
/// <li>
/// Setup the requisite accounting information for the
/// <see cref="Org.Apache.Hadoop.Mapreduce.Filecache.DistributedCache"/>
/// of the job, if necessary.
/// </li>
/// <li>
/// Copying the job's jar and configuration to the map-reduce system
/// directory on the distributed file-system.
/// </li>
/// <li>
/// Submitting the job to the <code>JobTracker</code> and optionally
/// monitoring it's status.
/// </li>
/// </ol></p>
/// </remarks>
/// <param name="job">the configuration to submit</param>
/// <param name="cluster">the handle to the Cluster</param>
/// <exception cref="System.TypeLoadException"/>
/// <exception cref="System.Exception"/>
/// <exception cref="System.IO.IOException"/>
internal virtual JobStatus SubmitJobInternal(Job job, Cluster cluster)
{
//validate the jobs output specs
CheckSpecs(job);
Configuration conf = job.GetConfiguration();
AddMRFrameworkToDistributedCache(conf);
Path jobStagingArea = JobSubmissionFiles.GetStagingDir(cluster, conf);
//configure the command line options correctly on the submitting dfs
IPAddress ip = Sharpen.Runtime.GetLocalHost();
if (ip != null)
{
submitHostAddress = ip.GetHostAddress();
submitHostName = ip.GetHostName();
conf.Set(MRJobConfig.JobSubmithost, submitHostName);
conf.Set(MRJobConfig.JobSubmithostaddr, submitHostAddress);
}
JobID jobId = submitClient.GetNewJobID();
job.SetJobID(jobId);
Path submitJobDir = new Path(jobStagingArea, jobId.ToString());
JobStatus status = null;
try
{
conf.Set(MRJobConfig.UserName, UserGroupInformation.GetCurrentUser().GetShortUserName
());
conf.Set("hadoop.http.filter.initializers", "org.apache.hadoop.yarn.server.webproxy.amfilter.AmFilterInitializer"
);
conf.Set(MRJobConfig.MapreduceJobDir, submitJobDir.ToString());
Log.Debug("Configuring job " + jobId + " with " + submitJobDir + " as the submit dir"
);
// get delegation token for the dir
TokenCache.ObtainTokensForNamenodes(job.GetCredentials(), new Path[] { submitJobDir }, conf);
PopulateTokenCache(conf, job.GetCredentials());
// generate a secret to authenticate shuffle transfers
if (TokenCache.GetShuffleSecretKey(job.GetCredentials()) == null)
{
KeyGenerator keyGen;
try
{
keyGen = KeyGenerator.GetInstance(ShuffleKeygenAlgorithm);
keyGen.Init(ShuffleKeyLength);
}
catch (NoSuchAlgorithmException e)
{
throw new IOException("Error generating shuffle secret key", e);
}
SecretKey shuffleKey = keyGen.GenerateKey();
TokenCache.SetShuffleSecretKey(shuffleKey.GetEncoded(), job.GetCredentials());
}
if (CryptoUtils.IsEncryptedSpillEnabled(conf))
{
conf.SetInt(MRJobConfig.MrAmMaxAttempts, 1);
Log.Warn("Max job attempts set to 1 since encrypted intermediate" + "data spill is enabled"
);
}
CopyAndConfigureFiles(job, submitJobDir);
Path submitJobFile = JobSubmissionFiles.GetJobConfPath(submitJobDir);
// Create the splits for the job
Log.Debug("Creating splits at " + jtFs.MakeQualified(submitJobDir));
int maps = WriteSplits(job, submitJobDir);
conf.SetInt(MRJobConfig.NumMaps, maps);
Log.Info("number of splits:" + maps);
// write "queue admins of the queue to which job is being submitted"
// to job file.
string queue = conf.Get(MRJobConfig.QueueName, JobConf.DefaultQueueName);
AccessControlList acl = submitClient.GetQueueAdmins(queue);
conf.Set(QueueManager.ToFullPropertyName(queue, QueueACL.AdministerJobs.GetAclName
()), acl.GetAclString());
// removing jobtoken referrals before copying the jobconf to HDFS
// as the tasks don't need this setting, actually they may break
// because of it if present as the referral will point to a
// different job.
TokenCache.CleanUpTokenReferral(conf);
if (conf.GetBoolean(MRJobConfig.JobTokenTrackingIdsEnabled, MRJobConfig.DefaultJobTokenTrackingIdsEnabled
))
{
// Add HDFS tracking ids
AList <string> trackingIds = new AList <string>();
foreach (Org.Apache.Hadoop.Security.Token.Token <TokenIdentifier> t in job.GetCredentials
().GetAllTokens())
{
trackingIds.AddItem(t.DecodeIdentifier().GetTrackingId());
}
conf.SetStrings(MRJobConfig.JobTokenTrackingIds, Sharpen.Collections.ToArray(trackingIds
, new string[trackingIds.Count]));
}
// Set reservation info if it exists
ReservationId reservationId = job.GetReservationId();
if (reservationId != null)
{
conf.Set(MRJobConfig.ReservationId, reservationId.ToString());
}
// Write job file to submit dir
WriteConf(conf, submitJobFile);
//
// Now, actually submit the job (using the submit name)
//
PrintTokens(jobId, job.GetCredentials());
status = submitClient.SubmitJob(jobId, submitJobDir.ToString(), job.GetCredentials
());
if (status != null)
{
return(status);
}
else
{
throw new IOException("Could not launch job");
}
}
finally
{
if (status == null)
{
Log.Info("Cleaning up the staging area " + submitJobDir);
if (jtFs != null && submitJobDir != null)
{
jtFs.Delete(submitJobDir, true);
}
}
}
}
public DelegationKey(int keyId, long expiryDate, SecretKey key)
: this(keyId, expiryDate, key != null ? key.GetEncoded() : null)
{
}
/// <exception cref="System.Exception"/>
internal virtual void TestRMAppStateStore(RMStateStoreTestBase.RMStateStoreHelper
stateStoreHelper, RMStateStoreTestBase.StoreStateVerifier verifier)
{
long submitTime = Runtime.CurrentTimeMillis();
long startTime = Runtime.CurrentTimeMillis() + 1234;
Configuration conf = new YarnConfiguration();
RMStateStore store = stateStoreHelper.GetRMStateStore();
RMStateStoreTestBase.TestDispatcher dispatcher = new RMStateStoreTestBase.TestDispatcher
();
store.SetRMDispatcher(dispatcher);
RMContext rmContext = Org.Mockito.Mockito.Mock <RMContext>();
Org.Mockito.Mockito.When(rmContext.GetStateStore()).ThenReturn(store);
AMRMTokenSecretManager appTokenMgr = Org.Mockito.Mockito.Spy(new AMRMTokenSecretManager
(conf, rmContext));
MasterKeyData masterKeyData = appTokenMgr.CreateNewMasterKey();
Org.Mockito.Mockito.When(appTokenMgr.GetMasterKey()).ThenReturn(masterKeyData);
ClientToAMTokenSecretManagerInRM clientToAMTokenMgr = new ClientToAMTokenSecretManagerInRM
();
ApplicationAttemptId attemptId1 = ConverterUtils.ToApplicationAttemptId("appattempt_1352994193343_0001_000001"
);
ApplicationId appId1 = attemptId1.GetApplicationId();
StoreApp(store, appId1, submitTime, startTime);
verifier.AfterStoreApp(store, appId1);
// create application token and client token key for attempt1
Org.Apache.Hadoop.Security.Token.Token <AMRMTokenIdentifier> appAttemptToken1 = GenerateAMRMToken
(attemptId1, appTokenMgr);
SecretKey clientTokenKey1 = clientToAMTokenMgr.CreateMasterKey(attemptId1);
ContainerId containerId1 = StoreAttempt(store, attemptId1, "container_1352994193343_0001_01_000001"
, appAttemptToken1, clientTokenKey1, dispatcher);
string appAttemptIdStr2 = "appattempt_1352994193343_0001_000002";
ApplicationAttemptId attemptId2 = ConverterUtils.ToApplicationAttemptId(appAttemptIdStr2
);
// create application token and client token key for attempt2
Org.Apache.Hadoop.Security.Token.Token <AMRMTokenIdentifier> appAttemptToken2 = GenerateAMRMToken
(attemptId2, appTokenMgr);
SecretKey clientTokenKey2 = clientToAMTokenMgr.CreateMasterKey(attemptId2);
ContainerId containerId2 = StoreAttempt(store, attemptId2, "container_1352994193343_0001_02_000001"
, appAttemptToken2, clientTokenKey2, dispatcher);
ApplicationAttemptId attemptIdRemoved = ConverterUtils.ToApplicationAttemptId("appattempt_1352994193343_0002_000001"
);
ApplicationId appIdRemoved = attemptIdRemoved.GetApplicationId();
StoreApp(store, appIdRemoved, submitTime, startTime);
StoreAttempt(store, attemptIdRemoved, "container_1352994193343_0002_01_000001", null
, null, dispatcher);
verifier.AfterStoreAppAttempt(store, attemptIdRemoved);
RMApp mockRemovedApp = Org.Mockito.Mockito.Mock <RMApp>();
RMAppAttemptMetrics mockRmAppAttemptMetrics = Org.Mockito.Mockito.Mock <RMAppAttemptMetrics
>();
Dictionary <ApplicationAttemptId, RMAppAttempt> attempts = new Dictionary <ApplicationAttemptId
, RMAppAttempt>();
ApplicationSubmissionContext context = new ApplicationSubmissionContextPBImpl();
context.SetApplicationId(appIdRemoved);
Org.Mockito.Mockito.When(mockRemovedApp.GetSubmitTime()).ThenReturn(submitTime);
Org.Mockito.Mockito.When(mockRemovedApp.GetApplicationSubmissionContext()).ThenReturn
(context);
Org.Mockito.Mockito.When(mockRemovedApp.GetAppAttempts()).ThenReturn(attempts);
Org.Mockito.Mockito.When(mockRemovedApp.GetUser()).ThenReturn("user1");
RMAppAttempt mockRemovedAttempt = Org.Mockito.Mockito.Mock <RMAppAttempt>();
Org.Mockito.Mockito.When(mockRemovedAttempt.GetAppAttemptId()).ThenReturn(attemptIdRemoved
);
Org.Mockito.Mockito.When(mockRemovedAttempt.GetRMAppAttemptMetrics()).ThenReturn(
mockRmAppAttemptMetrics);
Org.Mockito.Mockito.When(mockRmAppAttemptMetrics.GetAggregateAppResourceUsage()).
ThenReturn(new AggregateAppResourceUsage(0, 0));
attempts[attemptIdRemoved] = mockRemovedAttempt;
store.RemoveApplication(mockRemovedApp);
// remove application directory recursively.
StoreApp(store, appIdRemoved, submitTime, startTime);
StoreAttempt(store, attemptIdRemoved, "container_1352994193343_0002_01_000001", null
, null, dispatcher);
store.RemoveApplication(mockRemovedApp);
// let things settle down
Sharpen.Thread.Sleep(1000);
store.Close();
// give tester a chance to modify app state in the store
ModifyAppState();
// load state
store = stateStoreHelper.GetRMStateStore();
store.SetRMDispatcher(dispatcher);
RMStateStore.RMState state = store.LoadState();
IDictionary <ApplicationId, ApplicationStateData> rmAppState = state.GetApplicationState
();
ApplicationStateData appState = rmAppState[appId1];
// app is loaded
NUnit.Framework.Assert.IsNotNull(appState);
// app is loaded correctly
NUnit.Framework.Assert.AreEqual(submitTime, appState.GetSubmitTime());
NUnit.Framework.Assert.AreEqual(startTime, appState.GetStartTime());
// submission context is loaded correctly
NUnit.Framework.Assert.AreEqual(appId1, appState.GetApplicationSubmissionContext(
).GetApplicationId());
ApplicationAttemptStateData attemptState = appState.GetAttempt(attemptId1);
// attempt1 is loaded correctly
NUnit.Framework.Assert.IsNotNull(attemptState);
NUnit.Framework.Assert.AreEqual(attemptId1, attemptState.GetAttemptId());
NUnit.Framework.Assert.AreEqual(-1000, attemptState.GetAMContainerExitStatus());
// attempt1 container is loaded correctly
NUnit.Framework.Assert.AreEqual(containerId1, attemptState.GetMasterContainer().GetId
());
// attempt1 client token master key is loaded correctly
Assert.AssertArrayEquals(clientTokenKey1.GetEncoded(), attemptState.GetAppAttemptTokens
().GetSecretKey(RMStateStore.AmClientTokenMasterKeyName));
attemptState = appState.GetAttempt(attemptId2);
// attempt2 is loaded correctly
NUnit.Framework.Assert.IsNotNull(attemptState);
NUnit.Framework.Assert.AreEqual(attemptId2, attemptState.GetAttemptId());
// attempt2 container is loaded correctly
NUnit.Framework.Assert.AreEqual(containerId2, attemptState.GetMasterContainer().GetId
());
// attempt2 client token master key is loaded correctly
Assert.AssertArrayEquals(clientTokenKey2.GetEncoded(), attemptState.GetAppAttemptTokens
().GetSecretKey(RMStateStore.AmClientTokenMasterKeyName));
//******* update application/attempt state *******//
ApplicationStateData appState2 = ApplicationStateData.NewInstance(appState.GetSubmitTime
(), appState.GetStartTime(), appState.GetUser(), appState.GetApplicationSubmissionContext
(), RMAppState.Finished, "appDiagnostics", 1234);
appState2.attempts.PutAll(appState.attempts);
store.UpdateApplicationState(appState2);
ApplicationAttemptStateData oldAttemptState = attemptState;
ApplicationAttemptStateData newAttemptState = ApplicationAttemptStateData.NewInstance
(oldAttemptState.GetAttemptId(), oldAttemptState.GetMasterContainer(), oldAttemptState
.GetAppAttemptTokens(), oldAttemptState.GetStartTime(), RMAppAttemptState.Finished
, "myTrackingUrl", "attemptDiagnostics", FinalApplicationStatus.Succeeded, 100,
oldAttemptState.GetFinishTime(), 0, 0);
store.UpdateApplicationAttemptState(newAttemptState);
// test updating the state of an app/attempt whose initial state was not
// saved.
ApplicationId dummyAppId = ApplicationId.NewInstance(1234, 10);
ApplicationSubmissionContext dummyContext = new ApplicationSubmissionContextPBImpl
();
dummyContext.SetApplicationId(dummyAppId);
ApplicationStateData dummyApp = ApplicationStateData.NewInstance(appState.GetSubmitTime
(), appState.GetStartTime(), appState.GetUser(), dummyContext, RMAppState.Finished
, "appDiagnostics", 1234);
store.UpdateApplicationState(dummyApp);
ApplicationAttemptId dummyAttemptId = ApplicationAttemptId.NewInstance(dummyAppId
, 6);
ApplicationAttemptStateData dummyAttempt = ApplicationAttemptStateData.NewInstance
(dummyAttemptId, oldAttemptState.GetMasterContainer(), oldAttemptState.GetAppAttemptTokens
(), oldAttemptState.GetStartTime(), RMAppAttemptState.Finished, "myTrackingUrl",
"attemptDiagnostics", FinalApplicationStatus.Succeeded, 111, oldAttemptState.GetFinishTime
(), 0, 0);
store.UpdateApplicationAttemptState(dummyAttempt);
// let things settle down
Sharpen.Thread.Sleep(1000);
store.Close();
// check updated application state.
store = stateStoreHelper.GetRMStateStore();
store.SetRMDispatcher(dispatcher);
RMStateStore.RMState newRMState = store.LoadState();
IDictionary <ApplicationId, ApplicationStateData> newRMAppState = newRMState.GetApplicationState
();
NUnit.Framework.Assert.IsNotNull(newRMAppState[dummyApp.GetApplicationSubmissionContext
().GetApplicationId()]);
ApplicationStateData updatedAppState = newRMAppState[appId1];
NUnit.Framework.Assert.AreEqual(appState.GetApplicationSubmissionContext().GetApplicationId
(), updatedAppState.GetApplicationSubmissionContext().GetApplicationId());
NUnit.Framework.Assert.AreEqual(appState.GetSubmitTime(), updatedAppState.GetSubmitTime
());
NUnit.Framework.Assert.AreEqual(appState.GetStartTime(), updatedAppState.GetStartTime
());
NUnit.Framework.Assert.AreEqual(appState.GetUser(), updatedAppState.GetUser());
// new app state fields
NUnit.Framework.Assert.AreEqual(RMAppState.Finished, updatedAppState.GetState());
NUnit.Framework.Assert.AreEqual("appDiagnostics", updatedAppState.GetDiagnostics(
));
NUnit.Framework.Assert.AreEqual(1234, updatedAppState.GetFinishTime());
// check updated attempt state
NUnit.Framework.Assert.IsNotNull(newRMAppState[dummyApp.GetApplicationSubmissionContext
().GetApplicationId()].GetAttempt(dummyAttemptId));
ApplicationAttemptStateData updatedAttemptState = updatedAppState.GetAttempt(newAttemptState
.GetAttemptId());
NUnit.Framework.Assert.AreEqual(oldAttemptState.GetAttemptId(), updatedAttemptState
.GetAttemptId());
NUnit.Framework.Assert.AreEqual(containerId2, updatedAttemptState.GetMasterContainer
().GetId());
Assert.AssertArrayEquals(clientTokenKey2.GetEncoded(), attemptState.GetAppAttemptTokens
().GetSecretKey(RMStateStore.AmClientTokenMasterKeyName));
// new attempt state fields
NUnit.Framework.Assert.AreEqual(RMAppAttemptState.Finished, updatedAttemptState.GetState
());
NUnit.Framework.Assert.AreEqual("myTrackingUrl", updatedAttemptState.GetFinalTrackingUrl
());
NUnit.Framework.Assert.AreEqual("attemptDiagnostics", updatedAttemptState.GetDiagnostics
());
NUnit.Framework.Assert.AreEqual(100, updatedAttemptState.GetAMContainerExitStatus
());
NUnit.Framework.Assert.AreEqual(FinalApplicationStatus.Succeeded, updatedAttemptState
.GetFinalApplicationStatus());
// assert store is in expected state after everything is cleaned
NUnit.Framework.Assert.IsTrue(stateStoreHelper.IsFinalStateValid());
store.Close();
}
