Exemplo n.º 1
0
 /// <summary>
 /// Adds all the files referenced in the given JobSubmission
 /// </summary>
 /// <param name="submission"></param>
 internal void AddJobFiles(IJobSubmission submission)
 {
     AddToGlobalFiles(submission.GlobalFiles);
     AddToGlobalFiles(submission.GlobalAssemblies);
     AddToLocalFiles(submission.LocalFiles);
     AddToLocalFiles(submission.LocalAssemblies);
 }
Exemplo n.º 2
0
        private string CreateBootstrapAvroConfig(IJobSubmission jobSubmission, string driverFolder)
        {
            var paramInjector = TangFactory.GetTang().NewInjector(jobSubmission.DriverConfigurations.ToArray());

            var bootstrapArgs = new AvroJobSubmissionParameters
            {
                jobSubmissionFolder = driverFolder,
                jobId         = jobSubmission.JobIdentifier,
                tcpBeginPort  = paramInjector.GetNamedInstance <TcpPortRangeStart, int>(),
                tcpRangeCount = paramInjector.GetNamedInstance <TcpPortRangeCount, int>(),
                tcpTryCount   = paramInjector.GetNamedInstance <TcpPortRangeTryCount, int>(),
            };

            var avroLocalBootstrapArgs = new AvroLocalJobSubmissionParameters
            {
                sharedJobSubmissionParameters   = bootstrapArgs,
                maxNumberOfConcurrentEvaluators = _maxNumberOfConcurrentEvaluators
            };

            var submissionArgsFilePath = Path.Combine(driverFolder, _fileNames.GetJobSubmissionParametersFile());

            using (var argsFileStream = new FileStream(submissionArgsFilePath, FileMode.CreateNew))
            {
                var serializedArgs = AvroJsonSerializer <AvroLocalJobSubmissionParameters> .ToBytes(avroLocalBootstrapArgs);

                argsFileStream.Write(serializedArgs, 0, serializedArgs.Length);
            }

            return(submissionArgsFilePath);
        }
Exemplo n.º 3
0
        /// <summary>
        /// Creates the driver folder structure in this given folder as the root
        /// </summary>
        /// <param name="jobSubmission">Job submission information</param>
        /// <param name="driverFolderPath">Driver folder path</param>
        internal void CreateDefaultFolderStructure(IJobSubmission jobSubmission, string driverFolderPath)
        {
            Directory.CreateDirectory(Path.Combine(driverFolderPath, _fileNames.GetReefFolderName()));
            Directory.CreateDirectory(Path.Combine(driverFolderPath, _fileNames.GetLocalFolderPath()));
            Directory.CreateDirectory(Path.Combine(driverFolderPath, _fileNames.GetGlobalFolderPath()));

            var resourceHelper = new ResourceHelper(typeof(DriverFolderPreparationHelper).Assembly);

            foreach (var fileResources in clientFileResources)
            {
                var fileName = resourceHelper.GetString(fileResources.Item1);
                if (ClrDriverFullName == fileResources.Item1)
                {
                    fileName = Path.Combine(driverFolderPath, _fileNames.GetBridgeExePath());
                }
                File.WriteAllBytes(fileName, resourceHelper.GetBytes(fileResources.Item2));
            }

            var config = DefaultDriverConfigurationFileContents;

            if (!string.IsNullOrEmpty(jobSubmission.DriverConfigurationFileContents))
            {
                config = jobSubmission.DriverConfigurationFileContents;
            }
            File.WriteAllText(Path.Combine(driverFolderPath, _fileNames.GetBridgeExeConfigPath()), config);
        }
Exemplo n.º 4
0
        public void Submit(IJobSubmission jobSubmission)
        {
            var driverFolder           = PrepareDriverFolder(jobSubmission);
            var submissionArgsFilePath = CreateBootstrapAvroConfig(jobSubmission, driverFolder);

            _javaClientLauncher.Launch(JavaClassName, submissionArgsFilePath);
            Logger.Log(Level.Info, "Submitted the Driver for execution.");
        }
Exemplo n.º 5
0
        public void Submit(IJobSubmission jobSubmission)
        {
            // Prepare the job submission folder
            var driverFolderPath = CreateDriverFolder(jobSubmission.JobIdentifier);
            Logger.Log(Level.Info, "Preparing driver folder in " + driverFolderPath);

            Launch(jobSubmission, driverFolderPath);
        }
Exemplo n.º 6
0
        public void Submit(IJobSubmission jobSubmission)
        {
            // Prepare the job submission folder
            var driverFolderPath = CreateDriverFolder(jobSubmission.JobIdentifier);

            Logger.Log(Level.Info, "Preparing driver folder in " + driverFolderPath);

            Launch(jobSubmission, driverFolderPath);
        }
Exemplo n.º 7
0
 internal static JobParameters FromJobSubmission(IJobSubmission jobSubmission)
 {
     return new JobParameters(
         jobSubmission.JobIdentifier, 
         jobSubmission.MaxApplicationSubmissions, 
         jobSubmission.DriverMemory,
         null,
         null);
 }
Exemplo n.º 8
0
        /// <summary>
        /// Merges the Configurations in jobSubmission and serializes them into the right place within driverFolderPath,
        /// assuming
        /// that points to a Driver's working directory.
        /// </summary>
        /// <param name="jobSubmission"></param>
        /// <param name="driverFolderPath"></param>
        internal void CreateDriverConfiguration(IJobSubmission jobSubmission, string driverFolderPath)
        {
            var driverConfiguration = Configurations.Merge(jobSubmission.DriverConfigurations.ToArray());

            _configurationSerializer.ToFile(driverConfiguration,
                                            Path.Combine(driverFolderPath, _fileNames.GetClrDriverConfigurationPath()));

            // TODO: Remove once we cleaned up the Evaluator to not expect this [REEF-216]
            _configurationSerializer.ToFile(driverConfiguration,
                                            Path.Combine(driverFolderPath, _fileNames.GetGlobalFolderPath(), Constants.ClrBridgeRuntimeConfiguration));
        }
        /// <summary>
        /// Serializes the application parameters to reef/local/app-submission-params.json.
        /// </summary>
        internal void SerializeAppFile(IJobSubmission jobSubmission, IInjector paramInjector, string localDriverFolderPath)
        {
            var serializedArgs = SerializeAppArgsToBytes(jobSubmission, paramInjector, localDriverFolderPath);

            var submissionAppArgsFilePath = Path.Combine(
                localDriverFolderPath, _fileNames.GetLocalFolderPath(), _fileNames.GetAppSubmissionParametersFile());

            using (var jobArgsFileStream = new FileStream(submissionAppArgsFilePath, FileMode.CreateNew))
            {
                jobArgsFileStream.Write(serializedArgs, 0, serializedArgs.Length);
            }
        }
Exemplo n.º 10
0
        private string PrepareDriverFolder(IJobSubmission jobSubmission)
        {
            // Prepare the job submission folder
            var jobFolder    = CreateJobFolder(jobSubmission.JobIdentifier);
            var driverFolder = Path.Combine(jobFolder, DriverFolderName);

            Logger.Log(Level.Info, "Preparing driver folder in " + driverFolder);

            _driverFolderPreparationHelper.PrepareDriverFolder(jobSubmission, driverFolder);

            return(driverFolder);
        }
Exemplo n.º 11
0
        /// <summary>
        /// Serializes the application parameters to reef/local/app-submission-params.json.
        /// </summary>
        internal string SerializeAppFile(IJobSubmission jobSubmission, IInjector paramInjector, string driverFolderPath)
        {
            var serializedArgs = SerializeAppArgsToBytes(jobSubmission, paramInjector);

            var submissionArgsFilePath = Path.Combine(driverFolderPath, _fileNames.GetAppSubmissionParametersFile());
            using (var argsFileStream = new FileStream(submissionArgsFilePath, FileMode.CreateNew))
            {
                argsFileStream.Write(serializedArgs, 0, serializedArgs.Length);
            }

            return submissionArgsFilePath;
        }
Exemplo n.º 12
0
        /// <summary>
        /// Serializes the job parameters to job-submission-params.json.
        /// </summary>
        internal void SerializeJobFile(IJobSubmission jobSubmission, string localDriverFolderPath, string jobSubmissionDirectory)
        {
            var serializedArgs = SerializeJobArgsToBytes(jobSubmission, localDriverFolderPath, jobSubmissionDirectory);

            var submissionJobArgsFilePath = Path.Combine(localDriverFolderPath,
                _fileNames.GetJobSubmissionParametersFile());

            using (var jobArgsFileStream = new FileStream(submissionJobArgsFilePath, FileMode.CreateNew))
            {
                jobArgsFileStream.Write(serializedArgs, 0, serializedArgs.Length);
            }
        }
Exemplo n.º 13
0
        public IDriverHttpEndpoint SubmitAndGetDriverUrl(IJobSubmission jobSubmission)
        {
            // Prepare the job submission folder
            var driverFolderPath = CreateDriverFolder(jobSubmission.JobIdentifier);
            Logger.Log(Level.Info, "Preparing driver folder in " + driverFolderPath);

            Launch(jobSubmission, driverFolderPath);

            var pointerFileName = Path.Combine(driverFolderPath, _fileNames.DriverHttpEndpoint);

            var httpClient = new HttpClientHelper();
            _driverUrl = httpClient.GetDriverUrlForYarn(pointerFileName);

            return httpClient;
        }
Exemplo n.º 14
0
        public IJobSubmissionResult SubmitAndGetJobStatus(IJobSubmission jobSubmission)
        {
            var driverFolder           = PrepareDriverFolder(jobSubmission);
            var submissionArgsFilePath = CreateBootstrapAvroConfig(jobSubmission, driverFolder);

            Task.Run(() => _javaClientLauncher.Launch(JavaClassName, submissionArgsFilePath));

            var fileName = Path.Combine(driverFolder, _fileNames.DriverHttpEndpoint);
            JobSubmissionResult result = new LocalJobSubmissionResult(this, fileName);

            var msg = string.Format(CultureInfo.CurrentCulture,
                                    "Submitted the Driver for execution. Returned driverUrl is: {0}.", result.DriverUrl);

            Logger.Log(Level.Info, msg);
            return(result);
        }
Exemplo n.º 15
0
        private void Launch(IJobSubmission jobSubmission, string driverFolderPath)
        {
            _driverFolderPreparationHelper.PrepareDriverFolder(jobSubmission, driverFolderPath);

            // TODO: Remove this when we have a generalized way to pass config to java
            var paramInjector = TangFactory.GetTang().NewInjector(jobSubmission.DriverConfigurations.ToArray());

            var avroJobSubmissionParameters = new AvroJobSubmissionParameters
            {
                jobId               = jobSubmission.JobIdentifier,
                tcpBeginPort        = paramInjector.GetNamedInstance <TcpPortRangeStart, int>(),
                tcpRangeCount       = paramInjector.GetNamedInstance <TcpPortRangeCount, int>(),
                tcpTryCount         = paramInjector.GetNamedInstance <TcpPortRangeTryCount, int>(),
                jobSubmissionFolder = driverFolderPath
            };

            var avroYarnJobSubmissionParameters = new AvroYarnJobSubmissionParameters
            {
                driverMemory                  = jobSubmission.DriverMemory,
                driverRecoveryTimeout         = paramInjector.GetNamedInstance <DriverBridgeConfigurationOptions.DriverRestartEvaluatorRecoverySeconds, int>(),
                jobSubmissionDirectoryPrefix  = _jobSubmissionPrefix,
                sharedJobSubmissionParameters = avroJobSubmissionParameters
            };

            var avroYarnClusterJobSubmissionParameters = new AvroYarnClusterJobSubmissionParameters
            {
                maxApplicationSubmissions   = paramInjector.GetNamedInstance <DriverBridgeConfigurationOptions.MaxApplicationSubmissions, int>(),
                securityTokenKind           = _securityTokenKind,
                securityTokenService        = _securityTokenService,
                yarnJobSubmissionParameters = avroYarnJobSubmissionParameters
            };

            var submissionArgsFilePath = Path.Combine(driverFolderPath, _fileNames.GetJobSubmissionParametersFile());

            using (var argsFileStream = new FileStream(submissionArgsFilePath, FileMode.CreateNew))
            {
                var serializedArgs = AvroJsonSerializer <AvroYarnClusterJobSubmissionParameters> .ToBytes(avroYarnClusterJobSubmissionParameters);

                argsFileStream.Write(serializedArgs, 0, serializedArgs.Length);
            }

            // Submit the driver
            _javaClientLauncher.Launch(JavaClassName, submissionArgsFilePath);
            Logger.Log(Level.Info, "Submitted the Driver for execution." + jobSubmission.JobIdentifier);
        }
Exemplo n.º 16
0
        public IJobSubmissionResult SubmitAndGetJobStatus(IJobSubmission jobSubmission)
        {
            // Prepare the job submission folder
            var driverFolderPath = CreateDriverFolder(jobSubmission.JobIdentifier);
            Logger.Log(Level.Info, "Preparing driver folder in " + driverFolderPath);

            Launch(jobSubmission, driverFolderPath);

            var pointerFileName = Path.Combine(driverFolderPath, _fileNames.DriverHttpEndpoint);
            var jobSubmitionResultImpl = new YarnJobSubmissionResult(this, pointerFileName);

            var msg = string.Format(CultureInfo.CurrentCulture,
                "Submitted the Driver for execution. Returned driverUrl is: {0}, appId is {1}.",
                jobSubmitionResultImpl.DriverUrl, jobSubmitionResultImpl.AppId);
            Logger.Log(Level.Info, msg);

            return jobSubmitionResultImpl;
        }
Exemplo n.º 17
0
        internal byte[] SerializeAppArgsToBytes(IJobSubmission jobSubmission, IInjector paramInjector, string localDriverFolderPath)
        {
            var avroAppSubmissionParameters = new AvroAppSubmissionParameters
            {
                tcpBeginPort = paramInjector.GetNamedInstance<TcpPortRangeStart, int>(),
                tcpRangeCount = paramInjector.GetNamedInstance<TcpPortRangeCount, int>(),
                tcpTryCount = paramInjector.GetNamedInstance<TcpPortRangeTryCount, int>()
            };

            var avroYarnAppSubmissionParameters = new AvroYarnAppSubmissionParameters
            {
                sharedAppSubmissionParameters = avroAppSubmissionParameters,
                driverMemory = jobSubmission.DriverMemory,
                driverRecoveryTimeout =
                    paramInjector.GetNamedInstance<DriverBridgeConfigurationOptions.DriverRestartEvaluatorRecoverySeconds, int>(),
            };

            return AvroJsonSerializer<AvroYarnAppSubmissionParameters>.ToBytes(avroYarnAppSubmissionParameters);
        }
Exemplo n.º 18
0
        public IJobSubmissionResult SubmitAndGetJobStatus(IJobSubmission jobSubmission)
        {
            // Prepare the job submission folder
            var driverFolderPath = CreateDriverFolder(jobSubmission.JobIdentifier);

            Logger.Log(Level.Info, "Preparing driver folder in " + driverFolderPath);

            Launch(jobSubmission, driverFolderPath);

            var pointerFileName        = Path.Combine(driverFolderPath, _fileNames.DriverHttpEndpoint);
            var jobSubmitionResultImpl = new YarnJobSubmissionResult(this, pointerFileName);

            var msg = string.Format(CultureInfo.CurrentCulture,
                                    "Submitted the Driver for execution. Returned driverUrl is: {0}, appId is {1}.",
                                    jobSubmitionResultImpl.DriverUrl, jobSubmitionResultImpl.AppId);

            Logger.Log(Level.Info, msg);

            return(jobSubmitionResultImpl);
        }
Exemplo n.º 19
0
        private void Launch(IJobSubmission jobSubmission, string driverFolderPath)
        {
            _driverFolderPreparationHelper.PrepareDriverFolder(jobSubmission, driverFolderPath);

            //TODO: Remove this when we have a generalized way to pass config to java
            var javaParams = TangFactory.GetTang()
                .NewInjector(jobSubmission.DriverConfigurations.ToArray())
                .GetInstance<ClrClient2JavaClientCuratedParameters>();

            // Submit the driver
            _javaClientLauncher.Launch(JavaClassName, driverFolderPath, jobSubmission.JobIdentifier,
                jobSubmission.DriverMemory.ToString(),
                javaParams.TcpPortRangeStart.ToString(),
                javaParams.TcpPortRangeCount.ToString(),
                javaParams.TcpPortRangeTryCount.ToString(),
                javaParams.MaxApplicationSubmissions.ToString(),
                javaParams.DriverRestartEvaluatorRecoverySeconds.ToString()
                );
            Logger.Log(Level.Info, "Submitted the Driver for execution." + jobSubmission.JobIdentifier);
        }
        /// <summary>
        /// Prepares the working directory for a Driver in driverFolderPath.
        /// </summary>
        /// <param name="jobSubmission"></param>
        /// <param name="driverFolderPath"></param>
        internal void PrepareDriverFolder(IJobSubmission jobSubmission, string driverFolderPath)
        {
            Logger.Log(Level.Info, "Preparing Driver filesystem layout in " + driverFolderPath);

            // Setup the folder structure
            CreateDefaultFolderStructure(jobSubmission, driverFolderPath);

            // Add the jobSubmission into that folder structure
            _fileSets.AddJobFiles(jobSubmission);

            // Create the driver configuration
            CreateDriverConfiguration(jobSubmission, driverFolderPath);

            // Add the REEF assemblies
            AddAssemblies();

            // Initiate the final copy
            _fileSets.CopyToDriverFolder(driverFolderPath);

            Logger.Log(Level.Info, "Done preparing Driver filesystem layout in " + driverFolderPath);
        }
Exemplo n.º 21
0
        /// <summary>
        /// Prepares the working directory for a Driver in driverFolderPath.
        /// </summary>
        /// <param name="jobSubmission"></param>
        /// <param name="driverFolderPath"></param>
        internal void PrepareDriverFolder(IJobSubmission jobSubmission, string driverFolderPath)
        {
            Logger.Log(Level.Info, "Preparing Driver filesystem layout in " + driverFolderPath);

            // Setup the folder structure
            CreateDefaultFolderStructure(driverFolderPath);

            // Add the jobSubmission into that folder structure
            _fileSets.AddJobFiles(jobSubmission);

            // Create the driver configuration
            CreateDriverConfiguration(jobSubmission, driverFolderPath);

            // Add the REEF assemblies
            AddAssemblies();

            // Initiate the final copy
            _fileSets.CopyToDriverFolder(driverFolderPath);

            Logger.Log(Level.Info, "Done preparing Driver filesystem layout in " + driverFolderPath);
        }
Exemplo n.º 22
0
        public void Submit(IJobSubmission jobSubmission)
        {
            // Prepare the job submission folder
            var driverFolderPath = CreateDriverFolder(jobSubmission.JobIdentifier);
            Logger.Log(Level.Info, "Preparing driver folder in " + driverFolderPath);

            _driverFolderPreparationHelper.PrepareDriverFolder(jobSubmission, driverFolderPath);

            //TODO: Remove this when we have a generalized way to pass config to java
            var javaParams = TangFactory.GetTang()
                .NewInjector(jobSubmission.DriverConfigurations.ToArray())
                .GetInstance<ClrClient2JavaClientCuratedParameters>();

            // Submit the driver
            _javaClientLauncher.Launch(JavaClassName, driverFolderPath, jobSubmission.JobIdentifier,
                jobSubmission.DriverMemory.ToString(),
                javaParams.TcpPortRangeStart.ToString(),
                javaParams.TcpPortRangeCount.ToString(),
                javaParams.TcpPortRangeTryCount.ToString()
                );
            Logger.Log(Level.Info, "Submitted the Driver for execution.");
        }
Exemplo n.º 23
0
        public void Submit(IJobSubmission jobSubmission)
        {
            // Prepare the job submission folder
            var jobFolder    = CreateJobFolder(jobSubmission.JobIdentifier);
            var driverFolder = Path.Combine(jobFolder, DriverFolderName);

            Logger.Log(Level.Info, "Preparing driver folder in " + driverFolder);

            _driverFolderPreparationHelper.PrepareDriverFolder(jobSubmission, driverFolder);

            //TODO: Remove this when we have a generalized way to pass config to java
            var javaParams = TangFactory.GetTang()
                             .NewInjector(jobSubmission.DriverConfigurations.ToArray())
                             .GetInstance <ClrClient2JavaClientCuratedParameters>();

            _javaClientLauncher.Launch(JavaClassName, driverFolder, jobSubmission.JobIdentifier,
                                       _numberOfEvaluators.ToString(),
                                       javaParams.TcpPortRangeStart.ToString(),
                                       javaParams.TcpPortRangeCount.ToString(),
                                       javaParams.TcpPortRangeTryCount.ToString()
                                       );
            Logger.Log(Level.Info, "Submitted the Driver for execution.");
        }
Exemplo n.º 24
0
 public void Submit(IJobSubmission jobSubmission)
 {
     var driverFolder = PrepareDriverFolder(jobSubmission);
     var submissionArgsFilePath = CreateBootstrapAvroConfig(jobSubmission, driverFolder);
     _javaClientLauncher.Launch(JavaClassName, submissionArgsFilePath);
     Logger.Log(Level.Info, "Submitted the Driver for execution.");
 }
Exemplo n.º 25
0
        private string PrepareDriverFolder(IJobSubmission jobSubmission)
        {
            // Prepare the job submission folder
            var jobFolder = CreateJobFolder(jobSubmission.JobIdentifier);
            var driverFolder = Path.Combine(jobFolder, DriverFolderName);
            Logger.Log(Level.Info, "Preparing driver folder in " + driverFolder);

            _driverFolderPreparationHelper.PrepareDriverFolder(jobSubmission, driverFolder);

            return driverFolder;
        }
Exemplo n.º 26
0
        private void Launch(IJobSubmission jobSubmission, string driverFolderPath)
        {
            _driverFolderPreparationHelper.PrepareDriverFolder(jobSubmission, driverFolderPath);

            // TODO: Remove this when we have a generalized way to pass config to java
            var paramInjector = TangFactory.GetTang().NewInjector(jobSubmission.DriverConfigurations.ToArray());

            var submissionJobArgsFilePath = _paramSerializer.SerializeJobFile(jobSubmission, driverFolderPath);
            var submissionAppArgsFilePath = _paramSerializer.SerializeAppFile(jobSubmission, paramInjector, driverFolderPath);

            // Submit the driver
            _javaClientLauncher.Launch(JavaClassName, submissionJobArgsFilePath, submissionAppArgsFilePath);
            Logger.Log(Level.Info, "Submitted the Driver for execution." + jobSubmission.JobIdentifier);
        }
Exemplo n.º 27
0
 internal static JobRequest FromJobSubmission(IJobSubmission jobSubmission)
 {
     return new JobRequest(
         JobParameters.FromJobSubmission(jobSubmission), AppParameters.FromJobSubmission(jobSubmission));
 }
Exemplo n.º 28
0
        private SubmitApplication CreateApplicationSubmissionRequest(
           IJobSubmission jobSubmission,
           string appId,
           int maxApplicationSubmissions,
           JobResource jobResource)
        {
            string command = _yarnJobCommandProvider.GetJobSubmissionCommand();
            Log.Log(Level.Verbose, "Command for YARN: {0}", command);
            Log.Log(Level.Verbose, "ApplicationID: {0}", appId);
            Log.Log(Level.Verbose, "MaxApplicationSubmissions: {0}", maxApplicationSubmissions);
            Log.Log(Level.Verbose, "Driver archive location: {0}", jobResource.RemoteUploadPath);

            var submitApplication = new SubmitApplication
            {
                ApplicationId = appId,
                ApplicationName = jobSubmission.JobIdentifier,
                AmResource = new Resouce
                {
                    MemoryMB = jobSubmission.DriverMemory,
                    VCores = 1 // keeping parity with existing code
                },
                MaxAppAttempts = maxApplicationSubmissions,
                ApplicationType = REEFApplicationType,
                KeepContainersAcrossApplicationAttempts = true,
                Queue = @"default", // keeping parity with existing code
                Priority = 1, // keeping parity with existing code
                UnmanagedAM = false,
                AmContainerSpec = new AmContainerSpec
                {
                    LocalResources = new LocalResources
                    {
                        Entries = new[]
                        {
                            new KeyValuePair<string, LocalResourcesValue>
                            {
                                Key = _fileNames.GetReefFolderName(),
                                Value = new LocalResourcesValue
                                {
                                    Resource = jobResource.RemoteUploadPath,
                                    Type = ResourceType.ARCHIVE,
                                    Visibility = Visibility.APPLICATION,
                                    Size = jobResource.ResourceSize,
                                    Timestamp = jobResource.LastModificationUnixTimestamp
                                }
                            }
                        }
                    },
                    Commands = new Commands
                    {
                        Command = command
                    }
                }
            };

            return submitApplication;
        }
Exemplo n.º 29
0
        public void Submit(IJobSubmission jobSubmission)
        {
            string jobId = jobSubmission.JobIdentifier;

            // todo: Future client interface should be async.
            // Using GetAwaiter().GetResult() instead of .Result to avoid exception
            // getting wrapped in AggregateException.
            var newApplication = _yarnRMClient.CreateNewApplicationAsync().GetAwaiter().GetResult();
            string applicationId = newApplication.ApplicationId;

            // create job submission remote path
            string jobSubmissionDirectory =
                _jobSubmissionDirectoryProvider.GetJobSubmissionRemoteDirectory(applicationId);

            // create local driver folder.
            var localDriverFolderPath = CreateDriverFolder(jobId, applicationId);
            try
            {
                Log.Log(Level.Verbose, "Preparing driver folder in {0}", localDriverFolderPath);
                _driverFolderPreparationHelper.PrepareDriverFolder(jobSubmission, localDriverFolderPath);

                // prepare configuration
                var paramInjector = TangFactory.GetTang().NewInjector(jobSubmission.DriverConfigurations.ToArray());
                int maxApplicationSubmissions =
                    paramInjector.GetNamedInstance<DriverBridgeConfigurationOptions.MaxApplicationSubmissions, int>();

                var avroJobSubmissionParameters = new AvroJobSubmissionParameters
                {
                    jobId = jobSubmission.JobIdentifier,
                    tcpBeginPort = paramInjector.GetNamedInstance<TcpPortRangeStart, int>(),
                    tcpRangeCount = paramInjector.GetNamedInstance<TcpPortRangeCount, int>(),
                    tcpTryCount = paramInjector.GetNamedInstance<TcpPortRangeTryCount, int>(),
                    jobSubmissionFolder = localDriverFolderPath
                };

                var avroYarnJobSubmissionParameters = new AvroYarnJobSubmissionParameters
                {
                    driverMemory = jobSubmission.DriverMemory,
                    driverRecoveryTimeout =
                        paramInjector.GetNamedInstance<DriverBridgeConfigurationOptions.DriverRestartEvaluatorRecoverySeconds, int>(),
                    jobSubmissionDirectoryPrefix = jobSubmissionDirectory,
                    dfsJobSubmissionFolder = jobSubmissionDirectory,
                    sharedJobSubmissionParameters = avroJobSubmissionParameters
                };

                var submissionArgsFilePath = Path.Combine(localDriverFolderPath,
                    _fileNames.GetLocalFolderPath(),
                    _fileNames.GetJobSubmissionParametersFile());
                using (var argsFileStream = new FileStream(submissionArgsFilePath, FileMode.CreateNew))
                {
                    var serializedArgs =
                        AvroJsonSerializer<AvroYarnJobSubmissionParameters>.ToBytes(avroYarnJobSubmissionParameters);
                    argsFileStream.Write(serializedArgs, 0, serializedArgs.Length);
                }

                // upload prepared folder to DFS
                var jobResource = _jobResourceUploader.UploadJobResource(localDriverFolderPath, jobSubmissionDirectory);

                // submit job
                Log.Log(Level.Verbose, @"Assigned application id {0}", applicationId);

                var submissionReq = CreateApplicationSubmissionRequest(jobSubmission,
                    applicationId,
                    maxApplicationSubmissions,
                    jobResource);
                var submittedApplication = _yarnRMClient.SubmitApplicationAsync(submissionReq).GetAwaiter().GetResult();
                Log.Log(Level.Info, @"Submitted application {0}", submittedApplication.Id);
            }
            finally
            {
                if (Directory.Exists(localDriverFolderPath))
                {
                    Directory.Delete(localDriverFolderPath, recursive: true);
                }
            }
        }
Exemplo n.º 30
0
 public void Submit(IJobSubmission jobSubmission)
 {
     Submit(JobRequest.FromJobSubmission(jobSubmission));
 }
Exemplo n.º 31
0
        public IJobSubmissionResult SubmitAndGetJobStatus(IJobSubmission jobSubmission)
        {
            var driverFolder = PrepareDriverFolder(jobSubmission);
            var submissionArgsFilePath = CreateBootstrapAvroConfig(jobSubmission, driverFolder);

            Task.Run(() => _javaClientLauncher.Launch(JavaClassName, submissionArgsFilePath));

            var fileName = Path.Combine(driverFolder, _fileNames.DriverHttpEndpoint);
            JobSubmissionResult result = new LocalJobSubmissionResult(this, fileName);

            var msg = string.Format(CultureInfo.CurrentCulture,
                "Submitted the Driver for execution. Returned driverUrl is: {0}.", result.DriverUrl);
            Logger.Log(Level.Info,  msg);
            return result;
        }
        /// <summary>
        /// Creates the driver folder structure in this given folder as the root
        /// </summary>
        /// <param name="jobSubmission">Job submission information</param>
        /// <param name="driverFolderPath">Driver folder path</param>
        internal void CreateDefaultFolderStructure(IJobSubmission jobSubmission, string driverFolderPath)
        {
            Directory.CreateDirectory(Path.Combine(driverFolderPath, _fileNames.GetReefFolderName()));
            Directory.CreateDirectory(Path.Combine(driverFolderPath, _fileNames.GetLocalFolderPath()));
            Directory.CreateDirectory(Path.Combine(driverFolderPath, _fileNames.GetGlobalFolderPath()));

            var resourceHelper = new ResourceHelper(typeof(DriverFolderPreparationHelper).Assembly);
            foreach (var fileResources in clientFileResources)
            {
                var fileName = resourceHelper.GetString(fileResources.Item1);
                if (ClrDriverFullName == fileResources.Item1)
                {
                    fileName = Path.Combine(driverFolderPath, _fileNames.GetBridgeExePath());
                }
                File.WriteAllBytes(fileName, resourceHelper.GetBytes(fileResources.Item2));
            }
            
            var config = DefaultDriverConfigurationFileContents;
            if (!string.IsNullOrEmpty(jobSubmission.DriverConfigurationFileContents))
            {
                config = jobSubmission.DriverConfigurationFileContents;
            }
            File.WriteAllText(Path.Combine(driverFolderPath, _fileNames.GetBridgeExeConfigPath()), config);
        }
Exemplo n.º 33
0
 internal static AppParameters FromJobSubmission(IJobSubmission jobSubmission)
 {
     return(new AppParameters(jobSubmission.DriverConfigurations, jobSubmission.GlobalAssemblies, jobSubmission.GlobalFiles,
                              jobSubmission.LocalAssemblies, jobSubmission.LocalFiles, jobSubmission.DriverConfigurationFileContents));
 }
Exemplo n.º 34
0
 public IJobSubmissionResult SubmitAndGetJobStatus(IJobSubmission jobSubmission)
 {
     return(SubmitAndGetJobStatus(JobRequest.FromJobSubmission(jobSubmission)));
 }
Exemplo n.º 35
0
 internal static JobRequest FromJobSubmission(IJobSubmission jobSubmission)
 {
     return(new JobRequest(
                JobParameters.FromJobSubmission(jobSubmission), AppParameters.FromJobSubmission(jobSubmission)));
 }
Exemplo n.º 36
0
        private string CreateBootstrapAvroConfig(IJobSubmission jobSubmission, string driverFolder)
        {
            var paramInjector = TangFactory.GetTang().NewInjector(jobSubmission.DriverConfigurations.ToArray());

            var bootstrapArgs = new AvroJobSubmissionParameters
            {
                jobSubmissionFolder = driverFolder,
                jobId = jobSubmission.JobIdentifier,
                tcpBeginPort = paramInjector.GetNamedInstance<TcpPortRangeStart, int>(),
                tcpRangeCount = paramInjector.GetNamedInstance<TcpPortRangeCount, int>(),
                tcpTryCount = paramInjector.GetNamedInstance<TcpPortRangeTryCount, int>(),
            };

            var avroLocalBootstrapArgs = new AvroLocalJobSubmissionParameters
            {
                sharedJobSubmissionParameters = bootstrapArgs,
                maxNumberOfConcurrentEvaluators = _maxNumberOfConcurrentEvaluators
            };

            var submissionArgsFilePath = Path.Combine(driverFolder, _fileNames.GetJobSubmissionParametersFile());
            using (var argsFileStream = new FileStream(submissionArgsFilePath, FileMode.CreateNew))
            {
                var serializedArgs = AvroJsonSerializer<AvroLocalJobSubmissionParameters>.ToBytes(avroLocalBootstrapArgs);
                argsFileStream.Write(serializedArgs, 0, serializedArgs.Length);
            }

            return submissionArgsFilePath;
        }
Exemplo n.º 37
0
        private string CreateBootstrapAvroJobConfig(IJobSubmission jobSubmission, string driverFolder)
        {
            var bootstrapJobArgs = new AvroJobSubmissionParameters
            {
                jobSubmissionFolder = driverFolder,
                jobId = jobSubmission.JobIdentifier,
            };

            var submissionArgsFilePath = Path.Combine(driverFolder, _fileNames.GetJobSubmissionParametersFile());
            using (var argsFileStream = new FileStream(submissionArgsFilePath, FileMode.CreateNew))
            {
                var serializedArgs = AvroJsonSerializer<AvroJobSubmissionParameters>.ToBytes(bootstrapJobArgs);
                argsFileStream.Write(serializedArgs, 0, serializedArgs.Length);
            }

            return submissionArgsFilePath;
        }
Exemplo n.º 38
0
        internal byte[] SerializeJobArgsToBytes(IJobSubmission jobSubmission, string driverFolderPath)
        {
            var avroJobSubmissionParameters = new AvroJobSubmissionParameters
            {
                jobId = jobSubmission.JobIdentifier,
                jobSubmissionFolder = driverFolderPath
            };

            var avroYarnJobSubmissionParameters = new AvroYarnJobSubmissionParameters
            {
                jobSubmissionDirectoryPrefix = _jobSubmissionPrefix,
                sharedJobSubmissionParameters = avroJobSubmissionParameters
            };

            var avroYarnClusterJobSubmissionParameters = new AvroYarnClusterJobSubmissionParameters
            {
                securityTokenKind = _securityTokenKind,
                securityTokenService = _securityTokenService,
                yarnJobSubmissionParameters = avroYarnJobSubmissionParameters
            };

            return AvroJsonSerializer<AvroYarnClusterJobSubmissionParameters>.ToBytes(avroYarnClusterJobSubmissionParameters);
        }
Exemplo n.º 39
0
 public void Submit(IJobSubmission jobSubmission)
 {
     Submit(JobRequest.FromJobSubmission(jobSubmission));
 }
Exemplo n.º 40
0
        private void Launch(IJobSubmission jobSubmission, string driverFolderPath)
        {
            _driverFolderPreparationHelper.PrepareDriverFolder(jobSubmission, driverFolderPath);

            // TODO: Remove this when we have a generalized way to pass config to java
            var paramInjector = TangFactory.GetTang().NewInjector(jobSubmission.DriverConfigurations.ToArray());
                
            var avroJobSubmissionParameters = new AvroJobSubmissionParameters
            {
                jobId = jobSubmission.JobIdentifier,
                tcpBeginPort = paramInjector.GetNamedInstance<TcpPortRangeStart, int>(),
                tcpRangeCount = paramInjector.GetNamedInstance<TcpPortRangeCount, int>(),
                tcpTryCount = paramInjector.GetNamedInstance<TcpPortRangeTryCount, int>(),
                jobSubmissionFolder = driverFolderPath
            };

            var avroYarnJobSubmissionParameters = new AvroYarnJobSubmissionParameters
            {
                driverMemory = jobSubmission.DriverMemory,
                driverRecoveryTimeout = paramInjector.GetNamedInstance<DriverBridgeConfigurationOptions.DriverRestartEvaluatorRecoverySeconds, int>(),
                jobSubmissionDirectoryPrefix = _jobSubmissionPrefix,
                sharedJobSubmissionParameters = avroJobSubmissionParameters
            };

            var avroYarnClusterJobSubmissionParameters = new AvroYarnClusterJobSubmissionParameters
            {
                maxApplicationSubmissions = paramInjector.GetNamedInstance<DriverBridgeConfigurationOptions.MaxApplicationSubmissions, int>(),
                securityTokenKind = _securityTokenKind,
                securityTokenService = _securityTokenService,
                yarnJobSubmissionParameters = avroYarnJobSubmissionParameters
            };

            var submissionArgsFilePath = Path.Combine(driverFolderPath, _fileNames.GetJobSubmissionParametersFile());
            using (var argsFileStream = new FileStream(submissionArgsFilePath, FileMode.CreateNew))
            {
                var serializedArgs = AvroJsonSerializer<AvroYarnClusterJobSubmissionParameters>.ToBytes(avroYarnClusterJobSubmissionParameters);
                argsFileStream.Write(serializedArgs, 0, serializedArgs.Length);
            }

            // Submit the driver
            _javaClientLauncher.Launch(JavaClassName, submissionArgsFilePath);
            Logger.Log(Level.Info, "Submitted the Driver for execution." + jobSubmission.JobIdentifier);
        }
Exemplo n.º 41
0
 public IJobSubmissionResult SubmitAndGetJobStatus(IJobSubmission jobSubmission)
 {
     return SubmitAndGetJobStatus(JobRequest.FromJobSubmission(jobSubmission));
 }
Exemplo n.º 42
0
 internal static AppParameters FromJobSubmission(IJobSubmission jobSubmission)
 {
     return new AppParameters(jobSubmission.DriverConfigurations, jobSubmission.GlobalAssemblies, jobSubmission.GlobalFiles, 
         jobSubmission.LocalAssemblies, jobSubmission.LocalFiles, jobSubmission.DriverConfigurationFileContents);
 }
Exemplo n.º 43
0
 public IJobSubmissionResult SubmitAndGetJobStatus(IJobSubmission jobSubmission)
 {
     throw new NotSupportedException();
 }
Exemplo n.º 44
0
        internal byte[] SerializeJobArgsToBytes(IJobSubmission jobSubmission, string localDriverFolderPath, string jobSubmissionDirectory)
        {
            var avroJobSubmissionParameters = new AvroJobSubmissionParameters
            {
                jobId = jobSubmission.JobIdentifier,
                jobSubmissionFolder = localDriverFolderPath
            };

            var avroYarnJobSubmissionParameters = new AvroYarnJobSubmissionParameters
            {
                jobSubmissionDirectoryPrefix = jobSubmissionDirectory,
                dfsJobSubmissionFolder = jobSubmissionDirectory,
                sharedJobSubmissionParameters = avroJobSubmissionParameters
            };

            return AvroJsonSerializer<AvroYarnJobSubmissionParameters>.ToBytes(avroYarnJobSubmissionParameters);
        }
        /// <summary>
        /// Merges the Configurations in jobSubmission and serializes them into the right place within driverFolderPath,
        /// assuming
        /// that points to a Driver's working directory.
        /// </summary>
        /// <param name="jobSubmission"></param>
        /// <param name="driverFolderPath"></param>
        internal void CreateDriverConfiguration(IJobSubmission jobSubmission, string driverFolderPath)
        {
            var driverConfiguration = Configurations.Merge(jobSubmission.DriverConfigurations.ToArray());

            _configurationSerializer.ToFile(driverConfiguration,
                Path.Combine(driverFolderPath, _fileNames.GetClrDriverConfigurationPath()));

            // TODO: Remove once we cleaned up the Evaluator to not expect this [REEF-217]
            _configurationSerializer.ToFile(driverConfiguration,
                Path.Combine(driverFolderPath, _fileNames.GetGlobalFolderPath(), _fileNames.GetClrBridgeConfigurationName()));
        }
Exemplo n.º 46
0
        public IDriverHttpEndpoint SubmitAndGetDriverUrl(IJobSubmission jobSubmission)
        {
            // Prepare the job submission folder
            var jobFolder = CreateJobFolder(jobSubmission.JobIdentifier);
            var driverFolder = Path.Combine(jobFolder, DriverFolderName);
            Logger.Log(Level.Info, "Preparing driver folder in " + driverFolder);

            _driverFolderPreparationHelper.PrepareDriverFolder(jobSubmission, driverFolder);

            //TODO: Remove this when we have a generalized way to pass config to java
            var javaParams = TangFactory.GetTang()
                .NewInjector(jobSubmission.DriverConfigurations.ToArray())
                .GetInstance<ClrClient2JavaClientCuratedParameters>();

            Task.Run(() =>
            _javaClientLauncher.Launch(JavaClassName, driverFolder, jobSubmission.JobIdentifier,
                _numberOfEvaluators.ToString(),
                javaParams.TcpPortRangeStart.ToString(),
                javaParams.TcpPortRangeCount.ToString(),
                javaParams.TcpPortRangeTryCount.ToString()
                ));

            var fileName = Path.Combine(driverFolder, _fileNames.DriverHttpEndpoint);
            HttpClientHelper helper = new HttpClientHelper();
            _driverUrl = helper.GetDriverUrlForLocalRuntime(fileName);

            Logger.Log(Level.Info, "Submitted the Driver for execution. Returned driverUrl is: " + _driverUrl);
            return helper;
        }
Exemplo n.º 47
0
 internal static JobParameters FromJobSubmission(IJobSubmission jobSubmission)
 {
     return(new JobParameters(
                jobSubmission.JobIdentifier, jobSubmission.MaxApplicationSubmissions, jobSubmission.DriverMemory));
 }