Exemple #1
0
        public async Task TestSubmitNewApplication()
        {
            var ctx = new TestContext();
            var urlProvider = ctx.UrlProviderFake;
            var restReqExecutor = ctx.RestRequestExecutorFake;
            var anyUri = Enumerable.Repeat(new Uri("anyscheme://anypath"), 1);
            const string applicationId = "AnyApplicationId";
            const string anyApplicationType = "REEFTest";
            const string anyApplicationName = "AnyAPP";
            urlProvider.GetUrlAsync().Returns(Task.FromResult(anyUri));
            var anySubmitApplication = new SubmitApplication
            {
                ApplicationId = applicationId,
                AmResource = new Resouce
                {
                    MemoryMB = 500,
                    VCores = 1
                },
                ApplicationType = anyApplicationType,
                ApplicationName = anyApplicationName,
                KeepContainersAcrossApplicationAttempts = false,
                MaxAppAttempts = 1,
                Priority = 1,
                UnmanagedAM = false,
                AmContainerSpec = new AmContainerSpec
                {
                    Commands = new Commands
                    {
                        Command = @"DONTCARE"
                    },
                    LocalResources = new LocalResources
                    {
                        Entries = new List<YARN.RestClient.DataModel.KeyValuePair<string, LocalResourcesValue>>
                        {
                            new YARN.RestClient.DataModel.KeyValuePair<string, LocalResourcesValue>
                            {
                                Key = "APPLICATIONWILLFAILBUTWEDONTCAREHERE",
                                Value = new LocalResourcesValue
                                {
                                    Resource = "Foo",
                                    Type = ResourceType.FILE,
                                    Visibility = Visibility.APPLICATION
                                }
                            }
                        }
                    }
                }
            };

            const string expectedJson = @"{" +
                                            @"""application-id"":""AnyApplicationId""," +
                                            @"""application-name"":""AnyAPP""," +
                                            @"""queue"":null,""priority"":1," +
                                            @"""am-container-spec"":" +
                                            @"{" +
                                                @"""local-resources"":" +
                                                @"{" +
                                                    @"""entry"":" +
                                                    @"[" +
                                                        @"{" +
                                                            @"""key"":""APPLICATIONWILLFAILBUTWEDONTCAREHERE""," +
                                                            @"""value"":" +
                                                            @"{" +
                                                                @"""resource"":""Foo""," +
                                                                @"""type"":""FILE""," +
                                                                @"""visibility"":""APPLICATION""," +
                                                                @"""size"":0," +
                                                                @"""timestamp"":0" +
                                                            @"}" +
                                                        @"}" +
                                                    @"]" +
                                                @"}," + 
                                            @"""environment"":null," +
                                            @"""commands"":" +
                                            @"{" +
                                                @"""command"":""DONTCARE""" +
                                            @"}," +
                                            @"""service-data"":null," +
                                            @"""credentials"":null," +
                                            @"""application-acls"":null}," +
                                            @"""unmanaged-AM"":false," +
                                            @"""max-app-attempts"":1," +
                                            @"""resource"":" +
                                            @"{" +
                                                @"""memory"":500," +
                                                @"""vCores"":1" +
                                            @"},""application-type"":""REEFTest""," +
                                            @"""keep-containers-across-application-attempts"":false," +
                                            @"""application-tags"":null" +
                                        @"}";

            var thisApplication = new Application
            {
                AllocatedMB = 100,
                AmHostHttpAddress = "http://anyhttpaddress",
                AmContainerLogs = "SomeLogs",
                ApplicationType = "AnyYarnApplicationType",
                State = State.FINISHED,
                Name = "AnyApplicationName",
                RunningContainers = 0
            };

            var response = new RestResponse<VoidResult>
            {
                StatusCode = HttpStatusCode.Accepted
            };

            restReqExecutor.ExecuteAsync(
                Arg.Is<RestRequest>(
                    req =>
                        req.Resource == "ws/v1/cluster/apps"
                        && req.Method == Method.POST
                        && req.Content.Headers.ContentType.MediaType == "application/json"
                        && IsExpectedJson(req, expectedJson)),
                anyUri.First(),
                CancellationToken.None).Returns(Task.FromResult(response));

            restReqExecutor.ExecuteAsync<Application>(
                Arg.Is<RestRequest>(
                    req =>
                        req.Resource == "ws/v1/cluster/apps/" + applicationId
                        && req.RootElement == "app"
                        && req.Method == Method.GET),
                anyUri.First(),
                CancellationToken.None).Returns(Task.FromResult(thisApplication));

            var yarnClient = ctx.GetClient();
            Application actualApplication = await yarnClient.SubmitApplicationAsync(anySubmitApplication);

            Assert.Equal(thisApplication, actualApplication);
            var unused = urlProvider.Received(2).GetUrlAsync();
        }
        public async Task TestApplicationSubmissionAndQuery()
        {
            var client = TangFactory.GetTang().NewInjector().GetInstance<IYarnRMClient>();

            var newApplication = await client.CreateNewApplicationAsync();

            Assert.IsNotNull(newApplication);
            Assert.IsFalse(string.IsNullOrEmpty(newApplication.ApplicationId));
            Assert.IsTrue(newApplication.MaximumResourceCapability.MemoryMB > 0);
            Assert.IsTrue(newApplication.MaximumResourceCapability.VCores > 0);

            string applicationName = "REEFTEST_APPLICATION_" + Guid.NewGuid();
            Console.WriteLine(applicationName);

            const string anyApplicationType = "REEFTest";
            var submitApplicationRequest = new SubmitApplication
            {
                ApplicationId = newApplication.ApplicationId,
                AmResource = new Resouce
                {
                    MemoryMB = 500,
                    VCores = 1
                },
                ApplicationType = anyApplicationType,
                ApplicationName = applicationName,
                KeepContainersAcrossApplicationAttempts = false,
                MaxAppAttempts = 1,
                Priority = 1,
                UnmanagedAM = false,
                AmContainerSpec = new AmContainerSpec
                {
                    Commands = new Commands
                    {
                        Command = @"DONTCARE"
                    },
                    LocalResources = new LocalResources
                    {
                        Entries = new List<YARN.RestClient.DataModel.KeyValuePair<string, LocalResourcesValue>>
                        {
                            new YARN.RestClient.DataModel.KeyValuePair<string, LocalResourcesValue>
                            {
                                Key = "APPLICATIONWILLFAILBUTWEDONTCAREHERE",
                                Value = new LocalResourcesValue
                                {
                                    Resource = "Foo",
                                    Type = ResourceType.FILE,
                                    Visibility = Visibility.APPLICATION
                                }
                            }
                        }
                    }
                }
            };

            var application = await client.SubmitApplicationAsync(submitApplicationRequest);

            Assert.IsNotNull(application);
            Assert.AreEqual(newApplication.ApplicationId, application.Id);
            Assert.AreEqual(applicationName, application.Name);
            Assert.AreEqual(anyApplicationType, application.ApplicationType);

            var getApplicationResult = client.GetApplicationAsync(newApplication.ApplicationId).GetAwaiter().GetResult();

            Assert.IsNotNull(getApplicationResult);
            Assert.AreEqual(newApplication.ApplicationId, getApplicationResult.Id);
            Assert.AreEqual(applicationName, getApplicationResult.Name);
            Assert.AreEqual(anyApplicationType, getApplicationResult.ApplicationType);
        }
        private SubmitApplication CreateApplicationSubmissionRequest(
           IJobSubmission jobSubmission,
           string appId,
           int maxApplicationSubmissions,
           JobResource jobResource)
        {
            string command = _yarnJobCommandProvider.GetJobSubmissionCommand();
            Log.Log(Level.Verbose, "Command for YARN: {0}", command);
            Log.Log(Level.Verbose, "ApplicationID: {0}", appId);
            Log.Log(Level.Verbose, "MaxApplicationSubmissions: {0}", maxApplicationSubmissions);
            Log.Log(Level.Verbose, "Driver archive location: {0}", jobResource.RemoteUploadPath);

            var submitApplication = new SubmitApplication
            {
                ApplicationId = appId,
                ApplicationName = jobSubmission.JobIdentifier,
                AmResource = new Resouce
                {
                    MemoryMB = jobSubmission.DriverMemory,
                    VCores = 1 // keeping parity with existing code
                },
                MaxAppAttempts = maxApplicationSubmissions,
                ApplicationType = REEFApplicationType,
                KeepContainersAcrossApplicationAttempts = true,
                Queue = @"default", // keeping parity with existing code
                Priority = 1, // keeping parity with existing code
                UnmanagedAM = false,
                AmContainerSpec = new AmContainerSpec
                {
                    LocalResources = new LocalResources
                    {
                        Entries = new[]
                        {
                            new KeyValuePair<string, LocalResourcesValue>
                            {
                                Key = _fileNames.GetReefFolderName(),
                                Value = new LocalResourcesValue
                                {
                                    Resource = jobResource.RemoteUploadPath,
                                    Type = ResourceType.ARCHIVE,
                                    Visibility = Visibility.APPLICATION,
                                    Size = jobResource.ResourceSize,
                                    Timestamp = jobResource.LastModificationUnixTimestamp
                                }
                            }
                        }
                    },
                    Commands = new Commands
                    {
                        Command = command
                    }
                }
            };

            return submitApplication;
        }
        private SubmitApplication CreateApplicationSubmissionRequest(
           JobParameters jobParameters,
           string appId,
           int maxApplicationSubmissions,
           IReadOnlyCollection<JobResource> jobResources)
        {
            var commandProviderConfigModule = YarnCommandProviderConfiguration.ConfigurationModule;
            if (jobParameters.JavaLogLevel == JavaLoggingSetting.Verbose)
            {
                commandProviderConfigModule = commandProviderConfigModule
                    .Set(YarnCommandProviderConfiguration.JavaDebugLogging, true.ToString().ToLowerInvariant());
            }

            if (jobParameters.StdoutFilePath.IsPresent())
            {
                commandProviderConfigModule = commandProviderConfigModule
                    .Set(YarnCommandProviderConfiguration.DriverStdoutFilePath, jobParameters.StdoutFilePath.Value);
            }

            if (jobParameters.StderrFilePath.IsPresent())
            {
                commandProviderConfigModule = commandProviderConfigModule
                    .Set(YarnCommandProviderConfiguration.DriverStderrFilePath, jobParameters.StderrFilePath.Value);
            }

            var yarnJobCommandProvider = _injector.ForkInjector(commandProviderConfigModule.Build())
                .GetInstance<IYarnJobCommandProvider>();

            var command = yarnJobCommandProvider.GetJobSubmissionCommand();

            Log.Log(Level.Verbose, "Command for YARN: {0}", command);
            Log.Log(Level.Verbose, "ApplicationID: {0}", appId);
            Log.Log(Level.Verbose, "MaxApplicationSubmissions: {0}", maxApplicationSubmissions);
            foreach (var jobResource in jobResources)
            {
                Log.Log(Level.Verbose, "Remote file: {0}", jobResource.RemoteUploadPath);
            }

            var submitApplication = new SubmitApplication
            {
                ApplicationId = appId,
                ApplicationName = jobParameters.JobIdentifier,
                AmResource = new Resouce
                {
                    MemoryMB = jobParameters.DriverMemoryInMB,
                    VCores = 1 // keeping parity with existing code
                },
                MaxAppAttempts = maxApplicationSubmissions,
                ApplicationType = REEFApplicationType,
                KeepContainersAcrossApplicationAttempts = true,
                Queue = @"default", // keeping parity with existing code
                Priority = 1, // keeping parity with existing code
                UnmanagedAM = false,
                AmContainerSpec = new AmContainerSpec
                {
                    LocalResources = CreateLocalResources(jobResources),
                    Commands = new Commands
                    {
                        Command = command
                    }
                }
            };

            return submitApplication;
        }
        public async Task TestSubmitNewApplication()
        {
            var ctx = new TestContext();
            var urlProvider = ctx.UrlProviderFake;
            var restReqExecutor = ctx.RestRequestExecutorFake;
            Uri anyUri = new Uri("anyscheme://anypath");
            const string applicationId = "AnyApplicationId";
            const string anyApplicationType = "REEFTest";
            const string anyApplicationName = "AnyAPP";
            urlProvider.GetUrlAsync().Returns(Task.FromResult(anyUri));
            var anySubmitApplication = new SubmitApplication
            {
                ApplicationId = applicationId,
                AmResource = new Resouce
                {
                    MemoryMB = 500,
                    VCores = 1
                },
                ApplicationType = anyApplicationType,
                ApplicationName = anyApplicationName,
                KeepContainersAcrossApplicationAttempts = false,
                MaxAppAttempts = 1,
                Priority = 1,
                UnmanagedAM = false,
                AmContainerSpec = new AmContainerSpec
                {
                    Commands = new Commands
                    {
                        Command = @"DONTCARE"
                    },
                    LocalResources = new LocalResources
                    {
                        Entry = new List<KeyValuePair<string, LocalResourcesValue>>
                        {
                            new KeyValuePair<string, LocalResourcesValue>(
                                "APPLICATIONWILLFAILBUTWEDONTCAREHERE",
                                new LocalResourcesValue
                                {
                                    Resource = "Foo",
                                    Type = ResourceType.FILE,
                                    Visibility = Visibility.APPLICATION
                                })
                        }
                    }
                }
            };

            const string expectedJson = @"{" +
                                            @"""application-id"":""AnyApplicationId""," +
                                            @"""application-name"":""AnyAPP""," +
                                            @"""Queue"":null,""Priority"":1," +
                                            @"""am-container-spec"":" +
                                            @"{" +
                                                @"""local-resources"":" +
                                                @"{" +
                                                    @"""Entry"":" +
                                                    @"[" +
                                                        @"{" +
                                                            @"""Key"":""APPLICATIONWILLFAILBUTWEDONTCAREHERE""," +
                                                            @"""Value"":" +
                                                            @"{" +
                                                                @"""Resource"":""Foo""," +
                                                                @"""Type"":1," +
                                                                @"""Visibility"":2," +
                                                                @"""Size"":0," +
                                                                @"""Timestamp"":0" +
                                                            @"}" +
                                                        @"}" +
                                                    @"]" +
                                                @"}," + 
                                            @"""Environment"":null," +
                                            @"""Commands"":" +
                                            @"{" +
                                                @"""Command"":""DONTCARE""" +
                                            @"}," +
                                            @"""service-data"":null," +
                                            @"""Credentials"":null," +
                                            @"""application-acls"":null}," +
                                            @"""unmanaged-am"":false," +
                                            @"""max-app-attempts"":1," +
                                            @"""resource"":" +
                                            @"{" +
                                                @"""memory"":500," +
                                                @"""VCores"":1" +
                                            @"},""application-type"":""REEFTest""," +
                                            @"""keep-containers-across-application-attempts"":false," +
                                            @"""application-tags"":null" +
                                        @"}";

            var thisApplication = new Application
            {
                AllocatedMB = 100,
                AmHostHttpAddress = "http://anyhttpaddress",
                AmContainerLogs = "SomeLogs",
                ApplicationType = "AnyYarnApplicationType",
                State = State.FINISHED,
                Name = "AnyApplicationName",
                RunningContainers = 0
            };

            var response = Substitute.For<IRestResponse>();
            response.Headers.Returns(new List<Parameter>
            {
                new Parameter
                {
                    Name = "Location",
                    Value = "http://somelocation"
                }
            });
            response.StatusCode.Returns(HttpStatusCode.Accepted);

            restReqExecutor.ExecuteAsync(
                Arg.Is<IRestRequest>(
                    req =>
                        req.Resource == "ws/v1/cluster/apps"
                        && req.Method == Method.POST
                        && req.JsonSerializer is RestJsonSerializer
                        && req.Parameters.First().Name == "application/json"
                        && B(req, expectedJson)),
                anyUri,
                CancellationToken.None).Returns(Task.FromResult(response));

            restReqExecutor.ExecuteAsync<Application>(
                Arg.Is<IRestRequest>(
                    req =>
                        req.Resource == "ws/v1/cluster/apps/" + applicationId
                        && req.RootElement == "app"
                        && req.Method == Method.GET),
                anyUri,
                CancellationToken.None).Returns(Task.FromResult(thisApplication));

            var yarnClient = ctx.GetClient();
            Application actualApplication = await yarnClient.SubmitApplicationAsync(anySubmitApplication);

            Assert.AreEqual(thisApplication, actualApplication);
            var unused = urlProvider.Received(2).GetUrlAsync();
        }
 public async Task<Application> SubmitApplicationAsync(SubmitApplication submitApplicationRequest)
 {
     await new RemoveSynchronizationContextAwaiter();
     return await SubmitApplicationAsync(submitApplicationRequest, CancellationToken.None);
 }