internal PrivilegedAction <SubmitApplicationResponse> SetClientReq(ApplicationClientProtocol
client, SubmitApplicationRequest req)
{
this.client = client;
this.req = req;
return(this);
}
/// <exception cref="System.Exception"/>
private ApplicationId SubmitAppAndGetAppId(AccessControlList viewACL, AccessControlList
modifyACL)
{
SubmitApplicationRequest submitRequest = recordFactory.NewRecordInstance <SubmitApplicationRequest
>();
ApplicationSubmissionContext context = recordFactory.NewRecordInstance <ApplicationSubmissionContext
>();
ApplicationId applicationId = rmClient.GetNewApplication(recordFactory.NewRecordInstance
<GetNewApplicationRequest>()).GetApplicationId();
context.SetApplicationId(applicationId);
IDictionary <ApplicationAccessType, string> acls = new Dictionary <ApplicationAccessType
, string>();
acls[ApplicationAccessType.ViewApp] = viewACL.GetAclString();
acls[ApplicationAccessType.ModifyApp] = modifyACL.GetAclString();
ContainerLaunchContext amContainer = recordFactory.NewRecordInstance <ContainerLaunchContext
>();
Resource resource = BuilderUtils.NewResource(1024, 1);
context.SetResource(resource);
amContainer.SetApplicationACLs(acls);
context.SetAMContainerSpec(amContainer);
submitRequest.SetApplicationSubmissionContext(context);
rmClient.SubmitApplication(submitRequest);
resourceManager.WaitForState(applicationId, RMAppState.Accepted);
return(applicationId);
}
public async Task Then_Handler_Returns_True_If_Allowed_To_Submit()
{
ApplyRepository.Setup(r => r.CanSubmitApplication(It.IsAny <Guid>())).ReturnsAsync(true);
var request = new SubmitApplicationRequest
{
ApplicationId = Guid.NewGuid(),
SubmittingContactId = Guid.NewGuid(),
ApplyData = new ApplyData
{
ApplyDetails = new ApplyDetails(),
Sequences = new List <ApplySequence>
{
new ApplySequence
{
SequenceNo = 1,
Sections = new List <ApplySection>
{
new ApplySection
{
SectionNo = 1
}
}
}
}
}
};
var result = await Handler.Handle(request, new CancellationToken());
Assert.IsTrue(result);
ApplyRepository.Verify(r => r.SubmitApplication(It.IsAny <Guid>(), It.IsAny <ApplyData>(), It.IsAny <FinancialData>(), It.IsAny <Guid>()), Times.Once);
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
public virtual void Submit()
{
lock (this)
{
ApplicationSubmissionContext context = recordFactory.NewRecordInstance <ApplicationSubmissionContext
>();
context.SetApplicationId(this.applicationId);
context.SetQueue(this.queue);
// Set up the container launch context for the application master
ContainerLaunchContext amContainer = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord
<ContainerLaunchContext>();
context.SetAMContainerSpec(amContainer);
context.SetResource(Resources.CreateResource(YarnConfiguration.DefaultRmSchedulerMinimumAllocationMb
));
SubmitApplicationRequest request = recordFactory.NewRecordInstance <SubmitApplicationRequest
>();
request.SetApplicationSubmissionContext(context);
ResourceScheduler scheduler = resourceManager.GetResourceScheduler();
resourceManager.GetClientRMService().SubmitApplication(request);
// Notify scheduler
AppAddedSchedulerEvent addAppEvent = new AppAddedSchedulerEvent(this.applicationId
, this.queue, "user");
scheduler.Handle(addAppEvent);
AppAttemptAddedSchedulerEvent addAttemptEvent = new AppAttemptAddedSchedulerEvent
(this.applicationAttemptId, false);
scheduler.Handle(addAttemptEvent);
}
}
/// <exception cref="System.Exception"/>
private ApplicationId SubmitAppAndGetAppId(string submitter, string queueName, bool
setupACLs)
{
GetNewApplicationRequest newAppRequest = GetNewApplicationRequest.NewInstance();
ApplicationClientProtocol submitterClient = GetRMClientForUser(submitter);
ApplicationId applicationId = submitterClient.GetNewApplication(newAppRequest).GetApplicationId
();
Resource resource = BuilderUtils.NewResource(1024, 1);
IDictionary <ApplicationAccessType, string> acls = CreateACLs(submitter, setupACLs
);
ContainerLaunchContext amContainerSpec = ContainerLaunchContext.NewInstance(null,
null, null, null, null, acls);
ApplicationSubmissionContext appSubmissionContext = ApplicationSubmissionContext.
NewInstance(applicationId, "applicationName", queueName, null, amContainerSpec,
false, true, 1, resource, "applicationType");
appSubmissionContext.SetApplicationId(applicationId);
appSubmissionContext.SetQueue(queueName);
SubmitApplicationRequest submitRequest = SubmitApplicationRequest.NewInstance(appSubmissionContext
);
submitterClient.SubmitApplication(submitRequest);
resourceManager.WaitForState(applicationId, RMAppState.Accepted);
return(applicationId);
}
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
public override SubmitApplicationResponse SubmitApplication(SubmitApplicationRequest
request)
{
this._enclosing.ResetStartFailoverFlag(true);
// make sure failover has been triggered
NUnit.Framework.Assert.IsTrue(this._enclosing.WaittingForFailOver());
return(base.SubmitApplication(request));
}
/// <summary>
/// Submit an application to /v1/application/submit
/// </summary>
/// <param name="app">The application being submited to careerbuilder</param>
/// <returns></returns>
public ResponseApplication SubmitApplication(RequestApplication app)
{
var req = new SubmitApplicationRequest(_Settings);
WireBeforeRequestEvents(req);
WireAfterRequestEvents(req);
return(req.Submit(app));
}
public void Then_prevent_submission_if_another_user_has_already_submitted()
{
var request = new SubmitApplicationRequest {
ApplicationId = differentAppGuid, SubmittingContactId = differentAppGuid
};
Handler.Handle(request, new CancellationToken()).Wait();
ApplyRepository.Verify(r => r.SubmitApplication(It.IsAny <Guid>(), It.IsAny <ApplyData>(), It.IsAny <FinancialData>(), It.IsAny <Guid>()), Times.Never);
}
public async Task Then_Handler_Returns_False_If_Not_Allowed_To_Submit()
{
ApplyRepository.Setup(r => r.CanSubmitApplication(It.IsAny <Guid>())).ReturnsAsync(false);
var request = new SubmitApplicationRequest {
ApplicationId = Guid.NewGuid(), SubmittingContactId = Guid.NewGuid()
};
var result = await Handler.Handle(request, new CancellationToken());
Assert.IsFalse(result);
ApplyRepository.Verify(r => r.SubmitApplication(It.IsAny <Guid>(), It.IsAny <ApplyData>(), It.IsAny <FinancialData>(), It.IsAny <Guid>()), Times.Never);
}
public IActionResult Post(Guid applicationId, [FromBody] SubmitApplicationRequest action)
{
var command = new SubmitApplication
{
CorrelationId = Guid.NewGuid(),
ApplicationId = applicationId,
Comment = action.Comment
};
_bus.Send(command);
return(OaoResult.Accepted());
}
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
/// <exception cref="System.IO.IOException"/>
public virtual SubmitApplicationResponse SubmitApplication(SubmitApplicationRequest
request)
{
YarnServiceProtos.SubmitApplicationRequestProto requestProto = ((SubmitApplicationRequestPBImpl
)request).GetProto();
try
{
return(new SubmitApplicationResponsePBImpl(proxy.SubmitApplication(null, requestProto
)));
}
catch (ServiceException e)
{
RPCUtil.UnwrapAndThrowException(e);
return(null);
}
}
public void Then_allow_submission_if_user_is_the_one_whom_already_submitted()
{
var request = new SubmitApplicationRequest
{
ApplicationId = sameAppGuid, SubmittingContactId = sameAppGuid,
ApplyData = new ApplyData
{
ApplyDetails = new ApplyDetails(),
Sequences = new List <ApplySequence>()
}
};
Handler.Handle(request, new CancellationToken()).Wait();
ApplyRepository.Verify(r => r.SubmitApplication(It.IsAny <Guid>(), It.IsAny <ApplyData>(), It.IsAny <FinancialData>(), It.IsAny <Guid>()), Times.Once);
}
public virtual void Setup()
{
// start minicluster
conf = new YarnConfiguration();
yarnCluster = new MiniYARNCluster(typeof(TestAMRMClient).FullName, nodeCount, 1,
1);
yarnCluster.Init(conf);
yarnCluster.Start();
NUnit.Framework.Assert.IsNotNull(yarnCluster);
NUnit.Framework.Assert.AreEqual(Service.STATE.Started, yarnCluster.GetServiceState
());
// start rm client
yarnClient = (YarnClientImpl)YarnClient.CreateYarnClient();
yarnClient.Init(conf);
yarnClient.Start();
NUnit.Framework.Assert.IsNotNull(yarnClient);
NUnit.Framework.Assert.AreEqual(Service.STATE.Started, yarnClient.GetServiceState
());
// get node info
nodeReports = yarnClient.GetNodeReports(NodeState.Running);
// submit new app
ApplicationSubmissionContext appContext = yarnClient.CreateApplication().GetApplicationSubmissionContext
();
ApplicationId appId = appContext.GetApplicationId();
// set the application name
appContext.SetApplicationName("Test");
// Set the priority for the application master
Priority pri = Priority.NewInstance(0);
appContext.SetPriority(pri);
// Set the queue to which this application is to be submitted in the RM
appContext.SetQueue("default");
// Set up the container launch context for the application master
ContainerLaunchContext amContainer = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord
<ContainerLaunchContext>();
appContext.SetAMContainerSpec(amContainer);
// unmanaged AM
appContext.SetUnmanagedAM(true);
// Create the request to send to the applications manager
SubmitApplicationRequest appRequest = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord
<SubmitApplicationRequest>();
appRequest.SetApplicationSubmissionContext(appContext);
// Submit the application to the applications manager
yarnClient.SubmitApplication(appContext);
// wait for app to start
int iterationsLeft = 30;
RMAppAttempt appAttempt = null;
while (iterationsLeft > 0)
{
ApplicationReport appReport = yarnClient.GetApplicationReport(appId);
if (appReport.GetYarnApplicationState() == YarnApplicationState.Accepted)
{
attemptId = appReport.GetCurrentApplicationAttemptId();
appAttempt = yarnCluster.GetResourceManager().GetRMContext().GetRMApps()[attemptId
.GetApplicationId()].GetCurrentAppAttempt();
while (true)
{
if (appAttempt.GetAppAttemptState() == RMAppAttemptState.Launched)
{
break;
}
}
break;
}
Sleep(1000);
--iterationsLeft;
}
if (iterationsLeft == 0)
{
NUnit.Framework.Assert.Fail("Application hasn't bee started");
}
// Just dig into the ResourceManager and get the AMRMToken just for the sake
// of testing.
UserGroupInformation.SetLoginUser(UserGroupInformation.CreateRemoteUser(UserGroupInformation
.GetCurrentUser().GetUserName()));
UserGroupInformation.GetCurrentUser().AddToken(appAttempt.GetAMRMToken());
//creating an instance NMTokenCase
nmTokenCache = new NMTokenCache();
// start am rm client
rmClient = (AMRMClientImpl <AMRMClient.ContainerRequest>)AMRMClient.CreateAMRMClient
<AMRMClient.ContainerRequest>();
//setting an instance NMTokenCase
rmClient.SetNMTokenCache(nmTokenCache);
rmClient.Init(conf);
rmClient.Start();
NUnit.Framework.Assert.IsNotNull(rmClient);
NUnit.Framework.Assert.AreEqual(Service.STATE.Started, rmClient.GetServiceState()
);
// start am nm client
nmClient = (NMClientImpl)NMClient.CreateNMClient();
//propagating the AMRMClient NMTokenCache instance
nmClient.SetNMTokenCache(rmClient.GetNMTokenCache());
nmClient.Init(conf);
nmClient.Start();
NUnit.Framework.Assert.IsNotNull(nmClient);
NUnit.Framework.Assert.AreEqual(Service.STATE.Started, nmClient.GetServiceState()
);
}
public async Task <bool> SubmitApplication(SubmitApplicationRequest submitApplicationRequest)
{
return(await Post <SubmitApplicationRequest, bool>($"/Application/Submit", submitApplicationRequest));
}
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
/// <exception cref="System.IO.IOException"/>
public override ApplicationId SubmitApplication(ApplicationSubmissionContext appContext
)
{
ApplicationId applicationId = appContext.GetApplicationId();
if (applicationId == null)
{
throw new ApplicationIdNotProvidedException("ApplicationId is not provided in ApplicationSubmissionContext"
);
}
SubmitApplicationRequest request = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <
SubmitApplicationRequest>();
request.SetApplicationSubmissionContext(appContext);
// Automatically add the timeline DT into the CLC
// Only when the security and the timeline service are both enabled
if (IsSecurityEnabled() && timelineServiceEnabled)
{
AddTimelineDelegationToken(appContext.GetAMContainerSpec());
}
//TODO: YARN-1763:Handle RM failovers during the submitApplication call.
rmClient.SubmitApplication(request);
int pollCount = 0;
long startTime = Runtime.CurrentTimeMillis();
EnumSet <YarnApplicationState> waitingStates = EnumSet.Of(YarnApplicationState.New
, YarnApplicationState.NewSaving, YarnApplicationState.Submitted);
EnumSet <YarnApplicationState> failToSubmitStates = EnumSet.Of(YarnApplicationState
.Failed, YarnApplicationState.Killed);
while (true)
{
try
{
ApplicationReport appReport = GetApplicationReport(applicationId);
YarnApplicationState state = appReport.GetYarnApplicationState();
if (!waitingStates.Contains(state))
{
if (failToSubmitStates.Contains(state))
{
throw new YarnException("Failed to submit " + applicationId + " to YARN : " + appReport
.GetDiagnostics());
}
Log.Info("Submitted application " + applicationId);
break;
}
long elapsedMillis = Runtime.CurrentTimeMillis() - startTime;
if (EnforceAsyncAPITimeout() && elapsedMillis >= asyncApiPollTimeoutMillis)
{
throw new YarnException("Timed out while waiting for application " + applicationId
+ " to be submitted successfully");
}
// Notify the client through the log every 10 poll, in case the client
// is blocked here too long.
if (++pollCount % 10 == 0)
{
Log.Info("Application submission is not finished, " + "submitted application " +
applicationId + " is still in " + state);
}
try
{
Sharpen.Thread.Sleep(submitPollIntervalMillis);
}
catch (Exception)
{
Log.Error("Interrupted while waiting for application " + applicationId + " to be successfully submitted."
);
}
}
catch (ApplicationNotFoundException)
{
// FailOver or RM restart happens before RMStateStore saves
// ApplicationState
Log.Info("Re-submit application " + applicationId + "with the " + "same ApplicationSubmissionContext"
);
rmClient.SubmitApplication(request);
}
}
return(applicationId);
}
public async Task <ActionResult <bool> > Submit([FromBody] SubmitApplicationRequest request)
{
return(await _mediator.Send(request));
}
/// <exception cref="System.IO.IOException"/>
public virtual SubmitApplicationResponse SubmitApplication(SubmitApplicationRequest
request)
{
throw new IOException("Test");
}
public async Task Then_FinancialData_Is_Persisted()
{
ApplyRepository.Setup(r => r.CanSubmitApplication(It.IsAny <Guid>())).ReturnsAsync(true);
var request = new SubmitApplicationRequest
{
ApplicationId = Guid.NewGuid(),
SubmittingContactId = Guid.NewGuid(),
ApplyData = new ApplyData
{
ApplyDetails = new ApplyDetails(),
Sequences = new List <ApplySequence>
{
new ApplySequence
{
SequenceNo = 1,
Sections = new List <ApplySection>
{
new ApplySection
{
SectionNo = 1
}
}
}
}
},
FinancialData = new FinancialData
{
TurnOver = 1,
Depreciation = 2,
ProfitLoss = 3,
Dividends = 4,
IntangibleAssets = 5,
Assets = 6,
Liabilities = 7,
ShareholderFunds = 8,
Borrowings = 9,
AccountingReferenceDate = new DateTime(2021, 1, 1),
AccountingPeriod = 10,
AverageNumberofFTEEmployees = 11
}
};
var result = await Handler.Handle(request, new CancellationToken());
Assert.IsTrue(result);
ApplyRepository.Verify(r => r.SubmitApplication(It.IsAny <Guid>(), It.IsAny <ApplyData>(), It.Is <FinancialData>(
x => x.TurnOver == 1 &&
x.Depreciation == 2 &&
x.ProfitLoss == 3 &&
x.Dividends == 4 &&
x.IntangibleAssets == 5 &&
x.Assets == 6 &&
x.Liabilities == 7 &&
x.ShareholderFunds == 8 &&
x.Borrowings == 9 &&
x.AccountingReferenceDate == new DateTime(2021, 1, 1) &&
x.AccountingPeriod == 10 &&
x.AverageNumberofFTEEmployees == 11
), It.IsAny <Guid>()), Times.Once);
}
/// <exception cref="System.Exception"/>
public virtual RMApp SubmitApp(int masterMemory, string name, string user, IDictionary
<ApplicationAccessType, string> acls, bool unmanaged, string queue, int maxAppAttempts
, Credentials ts, string appType, bool waitForAccepted, bool keepContainers, bool
isAppIdProvided, ApplicationId applicationId, long attemptFailuresValidityInterval
, LogAggregationContext logAggregationContext, bool cancelTokensWhenComplete)
{
ApplicationId appId = isAppIdProvided ? applicationId : null;
ApplicationClientProtocol client = GetClientRMService();
if (!isAppIdProvided)
{
GetNewApplicationResponse resp = client.GetNewApplication(Org.Apache.Hadoop.Yarn.Util.Records
.NewRecord <GetNewApplicationRequest>());
appId = resp.GetApplicationId();
}
SubmitApplicationRequest req = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <SubmitApplicationRequest
>();
ApplicationSubmissionContext sub = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <
ApplicationSubmissionContext>();
sub.SetKeepContainersAcrossApplicationAttempts(keepContainers);
sub.SetApplicationId(appId);
sub.SetApplicationName(name);
sub.SetMaxAppAttempts(maxAppAttempts);
if (unmanaged)
{
sub.SetUnmanagedAM(true);
}
if (queue != null)
{
sub.SetQueue(queue);
}
sub.SetApplicationType(appType);
ContainerLaunchContext clc = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <ContainerLaunchContext
>();
Resource capability = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <Resource>();
capability.SetMemory(masterMemory);
sub.SetResource(capability);
clc.SetApplicationACLs(acls);
if (ts != null && UserGroupInformation.IsSecurityEnabled())
{
DataOutputBuffer dob = new DataOutputBuffer();
ts.WriteTokenStorageToStream(dob);
ByteBuffer securityTokens = ByteBuffer.Wrap(dob.GetData(), 0, dob.GetLength());
clc.SetTokens(securityTokens);
}
sub.SetAMContainerSpec(clc);
sub.SetAttemptFailuresValidityInterval(attemptFailuresValidityInterval);
if (logAggregationContext != null)
{
sub.SetLogAggregationContext(logAggregationContext);
}
sub.SetCancelTokensWhenComplete(cancelTokensWhenComplete);
req.SetApplicationSubmissionContext(sub);
UserGroupInformation fakeUser = UserGroupInformation.CreateUserForTesting(user, new
string[] { "someGroup" });
PrivilegedAction <SubmitApplicationResponse> action = new _PrivilegedAction_415().
SetClientReq(client, req);
fakeUser.DoAs(action);
// make sure app is immediately available after submit
if (waitForAccepted)
{
WaitForState(appId, RMAppState.Accepted);
}
RMApp rmApp = GetRMContext().GetRMApps()[appId];
// unmanaged AM won't go to RMAppAttemptState.SCHEDULED.
if (waitForAccepted && !unmanaged)
{
WaitForState(rmApp.GetCurrentAppAttempt().GetAppAttemptId(), RMAppAttemptState.Scheduled
);
}
return(rmApp);
}
