예제 #1
0
        /// <summary>
        /// Builds the command to be submitted to YARNRM
        /// </summary>
        /// <returns>Command string</returns>
        public string GetJobSubmissionCommand()
        {
            var sb = new StringBuilder();

            sb.Append(_fileNames.GetBridgeExePath());
            sb.Append(" " + JavaExe);
            sb.Append(" " + JvmOptionsPermSize);
            sb.Append(" " + string.Format(JvmOptionsMaxPermSizeFormat, _driverMaxPermSizeMB));
            sb.Append(" " +
                      string.Format(JvmOptionsMaxMemoryAllocationPoolSizeFormat, _driverMaxMemoryAllocationPoolSizeMB));
            sb.Append(" " + ClassPathToken);

            var yarnClasspathList = new List <string>(_yarnCommandLineEnvironment.GetYarnClasspathList())
            {
                string.Format("{0}/{1}/*", _fileNames.GetReefFolderName(), _fileNames.GetLocalFolderName()),
                string.Format("{0}/{1}/*", _fileNames.GetReefFolderName(), _fileNames.GetGlobalFolderName())
            };

            sb.Append(" " + string.Join(";", yarnClasspathList));
            sb.Append(" " + ProcReefProperty);
            if (_enableDebugLogging)
            {
                sb.Append(" " + JavaLoggingProperty);
            }

            sb.Append(" " + LauncherClassName);
            sb.Append(" " + _fileNames.GetJobSubmissionParametersFile());
            sb.Append(" " +
                      string.Format("{0}/{1}/{2}",
                                    _fileNames.GetReefFolderName(),
                                    _fileNames.GetLocalFolderName(),
                                    _fileNames.GetAppSubmissionParametersFile()));
            sb.Append(" " + _fileNames.GetDriverLoggingConfigCommand());
            return(sb.ToString());
        }
        /// <summary>
        /// Creates the driver folder structure in this given folder as the root
        /// </summary>
        /// <param name="appParameters">Job submission information</param>
        /// <param name="driverFolderPath">Driver folder path</param>
        internal void CreateDefaultFolderStructure(AppParameters appParameters, string driverFolderPath)
        {
            Directory.CreateDirectory(Path.Combine(driverFolderPath, _fileNames.GetReefFolderName()));
            Directory.CreateDirectory(Path.Combine(driverFolderPath, _fileNames.GetLocalFolderPath()));
            Directory.CreateDirectory(Path.Combine(driverFolderPath, _fileNames.GetGlobalFolderPath()));

            var resourceHelper = new ResourceHelper(typeof(DriverFolderPreparationHelper).Assembly);

            foreach (var fileResources in ResourceHelper.FileResources)
            {
                var fileName = resourceHelper.GetString(fileResources.Key);
                if (ResourceHelper.ClrDriverFullName == fileResources.Key)
                {
                    fileName = Path.Combine(driverFolderPath, _fileNames.GetBridgeExePath());
                }
                if (!File.Exists(fileName))
                {
                    File.WriteAllBytes(fileName, resourceHelper.GetBytes(fileResources.Value));
                }
            }

            // generate .config file for bridge executable
            var config = DefaultDriverConfigurationFileContents;

            if (!string.IsNullOrEmpty(appParameters.DriverConfigurationFileContents))
            {
                config = appParameters.DriverConfigurationFileContents;
            }
            File.WriteAllText(Path.Combine(driverFolderPath, _fileNames.GetBridgeExeConfigPath()), config);

            // generate .config file for Evaluator executable
            File.WriteAllText(Path.Combine(driverFolderPath, _fileNames.GetGlobalFolderPath(), EvaluatorExecutable),
                              DefaultDriverConfigurationFileContents);
        }
예제 #3
0
        /// <summary>
        /// Creates the driver folder structure in this given folder as the root
        /// </summary>
        /// <param name="jobSubmission">Job submission information</param>
        /// <param name="driverFolderPath">Driver folder path</param>
        internal void CreateDefaultFolderStructure(IJobSubmission jobSubmission, string driverFolderPath)
        {
            Directory.CreateDirectory(Path.Combine(driverFolderPath, _fileNames.GetReefFolderName()));
            Directory.CreateDirectory(Path.Combine(driverFolderPath, _fileNames.GetLocalFolderPath()));
            Directory.CreateDirectory(Path.Combine(driverFolderPath, _fileNames.GetGlobalFolderPath()));

            var resourceHelper = new ResourceHelper(typeof(DriverFolderPreparationHelper).Assembly);

            foreach (var fileResources in clientFileResources)
            {
                var fileName = resourceHelper.GetString(fileResources.Item1);
                if (ClrDriverFullName == fileResources.Item1)
                {
                    fileName = Path.Combine(driverFolderPath, _fileNames.GetBridgeExePath());
                }
                File.WriteAllBytes(fileName, resourceHelper.GetBytes(fileResources.Item2));
            }

            var config = DefaultDriverConfigurationFileContents;

            if (!string.IsNullOrEmpty(jobSubmission.DriverConfigurationFileContents))
            {
                config = jobSubmission.DriverConfigurationFileContents;
            }
            File.WriteAllText(Path.Combine(driverFolderPath, _fileNames.GetBridgeExeConfigPath()), config);
        }
예제 #4
0
        private void SerializeJobFile(string localDriverFolderPath, AvroAzureBatchJobSubmissionParameters jobParameters)
        {
            var serializedArgs = AvroJsonSerializer <AvroAzureBatchJobSubmissionParameters> .ToBytes(jobParameters);

            var submissionJobArgsFilePath = Path.Combine(localDriverFolderPath,
                                                         _fileNames.GetReefFolderName(), _fileNames.GetJobSubmissionParametersFile());

            using (var jobArgsFileStream = new FileStream(submissionJobArgsFilePath, FileMode.CreateNew))
            {
                jobArgsFileStream.Write(serializedArgs, 0, serializedArgs.Length);
            }
        }
예제 #5
0
        public string BuildDriverCommand(int driverMemory)
        {
            var sb = new StringBuilder();

            sb.Append(_fileNames.GetBridgeExePath())
            .Append(" " + JavaExe)
            .Append(" " + string.Format(JvmOptionsMaxMemoryAllocationPoolSizeFormat, driverMemory))
            .Append(" " + JvmOptionsPermSize)
            .Append(" " + JvmOptionsMaxPermSizeFormat)
            .Append(" " + ClassPathToken)
            .Append(" " + GetDriverClasspath())
            .Append(" " + ProcReefProperty)
            .Append(" " + LauncherClassName)
            .Append(" " + Path.Combine(_fileNames.GetReefFolderName(), _fileNames.GetJobSubmissionParametersFile()));
            return(string.Format(_osCommandFormat, _commandPrefix + sb.ToString()));
        }
예제 #6
0
        public string CreateArchiveToUpload(string folderPath)
        {
            string archivePath = Path.Combine(folderPath, Path.GetDirectoryName(folderPath) + ".zip");
            string reefFolder  = Path.Combine(folderPath, _reefFileNames.GetReefFolderName());

            if (!Directory.Exists(reefFolder))
            {
                throw new DirectoryNotFoundException("Cannot find directory " + reefFolder);
            }

            if (File.Exists(archivePath))
            {
                throw new InvalidOperationException("Archive file already exists " + archivePath);
            }

            ZipFile.CreateFromDirectory(reefFolder, archivePath);
            return(archivePath);
        }
예제 #7
0
 /// <summary>
 /// Creates the driver folder structure in this given folder as the root
 /// </summary>
 /// <param name="driverFolderPath"></param>
 internal void CreateDefaultFolderStructure(string driverFolderPath)
 {
     Directory.CreateDirectory(Path.Combine(driverFolderPath, _fileNames.GetReefFolderName()));
     Directory.CreateDirectory(Path.Combine(driverFolderPath, _fileNames.GetLocalFolderPath()));
     Directory.CreateDirectory(Path.Combine(driverFolderPath, _fileNames.GetGlobalFolderPath()));
 }
예제 #8
0
        public async Task <JobResource> UploadArchiveResourceAsync(string driverLocalFolderPath, string remoteUploadDirectoryPath)
        {
            driverLocalFolderPath = driverLocalFolderPath.TrimEnd('\\') + @"\";
            var driverUploadPath = remoteUploadDirectoryPath.TrimEnd('/') + @"/";

            Log.Log(Level.Info, "DriverFolderPath: {0} DriverUploadPath: {1}", driverLocalFolderPath, driverUploadPath);

            var archivePath = _resourceArchiveFileGenerator.CreateArchiveToUpload(driverLocalFolderPath);

            return(await UploadResourceAndGetInfoAsync(archivePath, ResourceType.ARCHIVE, driverUploadPath, _reefFileNames.GetReefFolderName()));
        }