public IActionResult Post(Guid applicationId, [FromBody] SubmitApplicationRequest action)
{
var command = new SubmitApplication
{
CorrelationId = Guid.NewGuid(),
ApplicationId = applicationId,
Comment = action.Comment
};
_bus.Send(command);
return OaoResult.Accepted();
}
public IActionResult Post(Guid applicationId, [FromBody] SubmitApplicationRequest action)
{
var command = new SubmitApplication
{
CorrelationId = Guid.NewGuid(),
ApplicationId = applicationId,
Comment = action.Comment
};
_bus.Send(command);
return(OaoResult.Accepted());
}
private SubmitApplication CreateApplicationSubmissionRequest(
JobParameters jobParameters,
string appId,
int maxApplicationSubmissions,
IReadOnlyCollection <JobResource> jobResources)
{
string command = _yarnJobCommandProvider.GetJobSubmissionCommand();
Log.Log(Level.Verbose, "Command for YARN: {0}", command);
Log.Log(Level.Verbose, "ApplicationID: {0}", appId);
Log.Log(Level.Verbose, "MaxApplicationSubmissions: {0}", maxApplicationSubmissions);
foreach (var jobResource in jobResources)
{
Log.Log(Level.Verbose, "Remote file: {0}", jobResource.RemoteUploadPath);
}
var submitApplication = new SubmitApplication
{
ApplicationId = appId,
ApplicationName = jobParameters.JobIdentifier,
AmResource = new Resouce
{
MemoryMB = jobParameters.DriverMemoryInMB,
VCores = 1 // keeping parity with existing code
},
MaxAppAttempts = maxApplicationSubmissions,
ApplicationType = REEFApplicationType,
KeepContainersAcrossApplicationAttempts = true,
Queue = @"default", // keeping parity with existing code
Priority = 1, // keeping parity with existing code
UnmanagedAM = false,
AmContainerSpec = new AmContainerSpec
{
LocalResources = CreateLocalResources(jobResources),
Commands = new Commands
{
Command = command
}
}
};
return(submitApplication);
}
public async Task <Application> SubmitApplicationAsync(
SubmitApplication submitApplication,
CancellationToken cancellationToken)
{
await new RemoveSynchronizationContextAwaiter();
var request = CreateRestRequest(SubmitApplication.Resource, Method.POST);
request.AddBody(submitApplication);
var submitResponse = await GenerateUrlAndExecuteRequestAsync(request, cancellationToken);
if (submitResponse.StatusCode != HttpStatusCode.Accepted)
{
throw new YarnRestAPIException(
string.Format("Application submission failed with HTTP STATUS {0}",
submitResponse.StatusCode));
}
return(await GetApplicationAsync(submitApplication.ApplicationId, cancellationToken));
}
private SubmitApplication CreateApplicationSubmissionRequest(
JobParameters jobParameters,
string appId,
int maxApplicationSubmissions,
IReadOnlyCollection <JobResource> jobResources)
{
var commandProviderConfigModule = YarnCommandProviderConfiguration.ConfigurationModule;
if (jobParameters.JavaLogLevel == JavaLoggingSetting.Verbose)
{
commandProviderConfigModule = commandProviderConfigModule
.Set(YarnCommandProviderConfiguration.JavaDebugLogging, true.ToString().ToLowerInvariant());
}
if (jobParameters.StdoutFilePath.IsPresent())
{
commandProviderConfigModule = commandProviderConfigModule
.Set(YarnCommandProviderConfiguration.DriverStdoutFilePath, jobParameters.StdoutFilePath.Value);
}
if (jobParameters.StderrFilePath.IsPresent())
{
commandProviderConfigModule = commandProviderConfigModule
.Set(YarnCommandProviderConfiguration.DriverStderrFilePath, jobParameters.StderrFilePath.Value);
}
var yarnJobCommandProvider = _injector.ForkInjector(commandProviderConfigModule.Build())
.GetInstance <IYarnJobCommandProvider>();
var command = yarnJobCommandProvider.GetJobSubmissionCommand();
Log.Log(Level.Verbose, "Command for YARN: {0}", command);
Log.Log(Level.Verbose, "ApplicationID: {0}", appId);
Log.Log(Level.Verbose, "MaxApplicationSubmissions: {0}", maxApplicationSubmissions);
foreach (var jobResource in jobResources)
{
Log.Log(Level.Verbose, "Remote file: {0}", jobResource.RemoteUploadPath);
}
var submitApplication = new SubmitApplication
{
ApplicationId = appId,
ApplicationName = jobParameters.JobIdentifier,
AmResource = new Resouce
{
MemoryMB = jobParameters.DriverMemoryInMB,
VCores = 1 // keeping parity with existing code
},
MaxAppAttempts = maxApplicationSubmissions,
ApplicationType = REEFApplicationType,
KeepContainersAcrossApplicationAttempts = true,
Queue = @"default", // keeping parity with existing code
Priority = 1, // keeping parity with existing code
UnmanagedAM = false,
AmContainerSpec = new AmContainerSpec
{
LocalResources = CreateLocalResources(jobResources),
Commands = new Commands
{
Command = command
}
}
};
return(submitApplication);
}
public async Task TestSubmitNewApplication()
{
var ctx = new TestContext();
var urlProvider = ctx.UrlProviderFake;
var restReqExecutor = ctx.RestRequestExecutorFake;
var anyUri = Enumerable.Repeat(new Uri("anyscheme://anypath"), 1);
const string applicationId = "AnyApplicationId";
const string anyApplicationType = "REEFTest";
const string anyApplicationName = "AnyAPP";
urlProvider.GetUrlAsync().Returns(Task.FromResult(anyUri));
var anySubmitApplication = new SubmitApplication
{
ApplicationId = applicationId,
AmResource = new Resouce
{
MemoryMB = 500,
VCores = 1
},
ApplicationType = anyApplicationType,
ApplicationName = anyApplicationName,
KeepContainersAcrossApplicationAttempts = false,
MaxAppAttempts = 1,
Priority = 1,
UnmanagedAM = false,
AmContainerSpec = new AmContainerSpec
{
Commands = new Commands
{
Command = @"DONTCARE"
},
LocalResources = new LocalResources
{
Entries = new List <YARN.RestClient.DataModel.KeyValuePair <string, LocalResourcesValue> >
{
new YARN.RestClient.DataModel.KeyValuePair <string, LocalResourcesValue>
{
Key = "APPLICATIONWILLFAILBUTWEDONTCAREHERE",
Value = new LocalResourcesValue
{
Resource = "Foo",
Type = ResourceType.FILE,
Visibility = Visibility.APPLICATION
}
}
}
}
}
};
const string expectedJson = @"{" +
@"""application-id"":""AnyApplicationId""," +
@"""application-name"":""AnyAPP""," +
@"""queue"":null,""priority"":1," +
@"""am-container-spec"":" +
@"{" +
@"""local-resources"":" +
@"{" +
@"""entry"":" +
@"[" +
@"{" +
@"""key"":""APPLICATIONWILLFAILBUTWEDONTCAREHERE""," +
@"""value"":" +
@"{" +
@"""resource"":""Foo""," +
@"""type"":""FILE""," +
@"""visibility"":""APPLICATION""," +
@"""size"":0," +
@"""timestamp"":0" +
@"}" +
@"}" +
@"]" +
@"}," +
@"""environment"":null," +
@"""commands"":" +
@"{" +
@"""command"":""DONTCARE""" +
@"}," +
@"""service-data"":null," +
@"""credentials"":null," +
@"""application-acls"":null}," +
@"""unmanaged-AM"":false," +
@"""max-app-attempts"":1," +
@"""resource"":" +
@"{" +
@"""memory"":500," +
@"""vCores"":1" +
@"},""application-type"":""REEFTest""," +
@"""keep-containers-across-application-attempts"":false," +
@"""application-tags"":null" +
@"}";
var thisApplication = new Application
{
AllocatedMB = 100,
AmHostHttpAddress = "http://anyhttpaddress",
AmContainerLogs = "SomeLogs",
ApplicationType = "AnyYarnApplicationType",
State = State.FINISHED,
Name = "AnyApplicationName",
RunningContainers = 0
};
var response = new RestResponse <VoidResult>
{
StatusCode = HttpStatusCode.Accepted
};
restReqExecutor.ExecuteAsync(
Arg.Is <RestRequest>(
req =>
req.Resource == "ws/v1/cluster/apps" &&
req.Method == Method.POST &&
req.Content.Headers.ContentType.MediaType == "application/json" &&
IsExpectedJson(req, expectedJson)),
anyUri.First(),
CancellationToken.None).Returns(Task.FromResult(response));
restReqExecutor.ExecuteAsync <Application>(
Arg.Is <RestRequest>(
req =>
req.Resource == "ws/v1/cluster/apps/" + applicationId &&
req.RootElement == "app" &&
req.Method == Method.GET),
anyUri.First(),
CancellationToken.None).Returns(Task.FromResult(thisApplication));
var yarnClient = ctx.GetClient();
Application actualApplication = await yarnClient.SubmitApplicationAsync(anySubmitApplication);
Assert.Equal(thisApplication, actualApplication);
var unused = urlProvider.Received(2).GetUrlAsync();
}
public async Task TestApplicationSubmissionAndQuery()
{
var injector = TangFactory.GetTang().NewInjector();
var cred = new HDInsightTestCredential();
var urlProvider = new HDInsightRMUrlProvider();
injector.BindVolatileInstance(GenericType <IYarnRestClientCredential> .Class, cred);
injector.BindVolatileInstance(GenericType <IUrlProvider> .Class, urlProvider);
var client = injector.GetInstance <IYarnRMClient>();
var newApplication = await client.CreateNewApplicationAsync();
Assert.NotNull(newApplication);
Assert.False(string.IsNullOrEmpty(newApplication.ApplicationId));
Assert.True(newApplication.MaximumResourceCapability.MemoryMB > 0);
Assert.True(newApplication.MaximumResourceCapability.VCores > 0);
string applicationName = "REEFTEST_APPLICATION_" + Guid.NewGuid();
Console.WriteLine(applicationName);
const string anyApplicationType = "REEFTest";
var submitApplicationRequest = new SubmitApplication
{
ApplicationId = newApplication.ApplicationId,
AmResource = new Resouce
{
MemoryMB = 500,
VCores = 1
},
ApplicationType = anyApplicationType,
ApplicationName = applicationName,
KeepContainersAcrossApplicationAttempts = false,
MaxAppAttempts = 1,
Priority = 1,
UnmanagedAM = false,
AmContainerSpec = new AmContainerSpec
{
Commands = new Commands
{
Command = @"DONTCARE"
},
LocalResources = new LocalResources
{
Entries = new List <YARN.RestClient.DataModel.KeyValuePair <string, LocalResourcesValue> >
{
new YARN.RestClient.DataModel.KeyValuePair <string, LocalResourcesValue>
{
Key = "APPLICATIONWILLFAILBUTWEDONTCAREHERE",
Value = new LocalResourcesValue
{
Resource = "Foo",
Type = ResourceType.FILE,
Visibility = Visibility.APPLICATION
}
}
}
}
}
};
var application = await client.SubmitApplicationAsync(submitApplicationRequest);
Assert.NotNull(application);
Assert.Equal(newApplication.ApplicationId, application.Id);
Assert.Equal(applicationName, application.Name);
Assert.Equal(anyApplicationType, application.ApplicationType);
var getApplicationResult = client.GetApplicationAsync(newApplication.ApplicationId).GetAwaiter().GetResult();
Assert.NotNull(getApplicationResult);
Assert.Equal(newApplication.ApplicationId, getApplicationResult.Id);
Assert.Equal(applicationName, getApplicationResult.Name);
Assert.Equal(anyApplicationType, getApplicationResult.ApplicationType);
}
public async Task <Application> SubmitApplicationAsync(SubmitApplication submitApplicationRequest)
{
await new RemoveSynchronizationContextAwaiter();
return(await SubmitApplicationAsync(submitApplicationRequest, CancellationToken.None));
}