/// <summary> /// Serializes the application parameters to reef/local/app-submission-params.json. /// </summary> internal void SerializeAppFile(AppParameters appParameters, IInjector paramInjector, string localDriverFolderPath) { var serializedArgs = SerializeAppArgsToBytes(appParameters, paramInjector, localDriverFolderPath); var submissionAppArgsFilePath = Path.Combine( localDriverFolderPath, _fileNames.GetLocalFolderPath(), _fileNames.GetAppSubmissionParametersFile()); using (var jobArgsFileStream = new FileStream(submissionAppArgsFilePath, FileMode.CreateNew)) { jobArgsFileStream.Write(serializedArgs, 0, serializedArgs.Length); } }
/// <summary> /// Serializes the application parameters to reef/local/app-submission-params.json. /// </summary> internal string SerializeAppFile(AppParameters appParameters, IInjector paramInjector, string driverFolderPath) { var serializedArgs = SerializeAppArgsToBytes(appParameters, paramInjector); var submissionArgsFilePath = Path.Combine(driverFolderPath, _fileNames.GetAppSubmissionParametersFile()); using (var argsFileStream = new FileStream(submissionArgsFilePath, FileMode.CreateNew)) { argsFileStream.Write(serializedArgs, 0, serializedArgs.Length); } return submissionArgsFilePath; }
internal byte[] SerializeAppArgsToBytes(AppParameters appParameters, IInjector paramInjector) { var avroAppSubmissionParameters = new AvroAppSubmissionParameters { tcpBeginPort = paramInjector.GetNamedInstance<TcpPortRangeStart, int>(), tcpRangeCount = paramInjector.GetNamedInstance<TcpPortRangeCount, int>(), tcpTryCount = paramInjector.GetNamedInstance<TcpPortRangeTryCount, int>() }; var avroYarnAppSubmissionParameters = new AvroYarnAppSubmissionParameters { sharedAppSubmissionParameters = avroAppSubmissionParameters, driverRecoveryTimeout = paramInjector.GetNamedInstance<DriverBridgeConfigurationOptions.DriverRestartEvaluatorRecoverySeconds, int>() }; return AvroJsonSerializer<AvroYarnAppSubmissionParameters>.ToBytes(avroYarnAppSubmissionParameters); }
/// <summary> /// Prepares the working directory for a Driver in driverFolderPath. /// </summary> /// <param name="appParameters"></param> /// <param name="driverFolderPath"></param> internal void PrepareDriverFolder(AppParameters appParameters, string driverFolderPath) { Logger.Log(Level.Verbose, "Preparing Driver filesystem layout in " + driverFolderPath); // Setup the folder structure CreateDefaultFolderStructure(appParameters, driverFolderPath); // Add the appParameters into that folder structure _fileSets.AddJobFiles(appParameters); // Create the driver configuration CreateDriverConfiguration(appParameters, driverFolderPath); // Add the REEF assemblies AddAssemblies(); // Initiate the final copy _fileSets.CopyToDriverFolder(driverFolderPath); Logger.Log(Level.Info, "Done preparing Driver filesystem layout in " + driverFolderPath); }
internal static JobRequest FromJobSubmission(IJobSubmission jobSubmission) { return(new JobRequest( JobParameters.FromJobSubmission(jobSubmission), AppParameters.FromJobSubmission(jobSubmission))); }
internal JobRequest(JobParameters jobParameters, AppParameters appParameters) { _jobParameters = jobParameters; _appParameters = appParameters; }
/// <summary> /// Creates the driver folder structure in this given folder as the root /// </summary> /// <param name="appParameters">Job submission information</param> /// <param name="driverFolderPath">Driver folder path</param> internal void CreateDefaultFolderStructure(AppParameters appParameters, string driverFolderPath) { Directory.CreateDirectory(Path.Combine(driverFolderPath, _fileNames.GetReefFolderName())); Directory.CreateDirectory(Path.Combine(driverFolderPath, _fileNames.GetLocalFolderPath())); Directory.CreateDirectory(Path.Combine(driverFolderPath, _fileNames.GetGlobalFolderPath())); var resourceHelper = new ResourceHelper(typeof(DriverFolderPreparationHelper).Assembly); foreach (var fileResources in ResourceHelper.FileResources) { var fileName = resourceHelper.GetString(fileResources.Key); if (ResourceHelper.ClrDriverFullName == fileResources.Key) { fileName = Path.Combine(driverFolderPath, _fileNames.GetBridgeExePath()); } if (!File.Exists(fileName)) { File.WriteAllBytes(fileName, resourceHelper.GetBytes(fileResources.Value)); } } // generate .config file for bridge executable var config = DefaultDriverConfigurationFileContents; if (!string.IsNullOrEmpty(appParameters.DriverConfigurationFileContents)) { config = appParameters.DriverConfigurationFileContents; } File.WriteAllText(Path.Combine(driverFolderPath, _fileNames.GetBridgeExeConfigPath()), config); // generate .config file for Evaluator executable File.WriteAllText(Path.Combine(driverFolderPath, _fileNames.GetGlobalFolderPath(), EvaluatorExecutable), DefaultDriverConfigurationFileContents); }
/// <summary> /// Merges the Configurations in appParameters and serializes them into the right place within driverFolderPath, /// assuming /// that points to a Driver's working directory. /// </summary> /// <param name="appParameters"></param> /// <param name="driverFolderPath"></param> internal void CreateDriverConfiguration(AppParameters appParameters, string driverFolderPath) { var driverConfigurations = _driverConfigurationProviders.Select(configurationProvider => configurationProvider.GetConfiguration()).ToList(); var driverConfiguration = Configurations.Merge(driverConfigurations.Concat(appParameters.DriverConfigurations).ToArray()); _configurationSerializer.ToFile(driverConfiguration, Path.Combine(driverFolderPath, _fileNames.GetClrDriverConfigurationPath())); }
private string CreateBootstrapAvroAppConfig(AppParameters appParameters, string driverFolder) { var paramInjector = TangFactory.GetTang().NewInjector(appParameters.DriverConfigurations.ToArray()); var bootstrapAppArgs = new AvroAppSubmissionParameters { tcpBeginPort = paramInjector.GetNamedInstance<TcpPortRangeStart, int>(), tcpRangeCount = paramInjector.GetNamedInstance<TcpPortRangeCount, int>(), tcpTryCount = paramInjector.GetNamedInstance<TcpPortRangeTryCount, int>(), }; var avroLocalBootstrapAppArgs = new AvroLocalAppSubmissionParameters { sharedAppSubmissionParameters = bootstrapAppArgs, maxNumberOfConcurrentEvaluators = _maxNumberOfConcurrentEvaluators }; var submissionArgsFilePath = Path.Combine(driverFolder, _fileNames.GetAppSubmissionParametersFile()); using (var argsFileStream = new FileStream(submissionArgsFilePath, FileMode.CreateNew)) { var serializedArgs = AvroJsonSerializer<AvroLocalAppSubmissionParameters>.ToBytes(avroLocalBootstrapAppArgs); argsFileStream.Write(serializedArgs, 0, serializedArgs.Length); } return submissionArgsFilePath; }