private JobJarMaker( IResourceArchiveFileGenerator resourceArchiveFileGenerator, DriverFolderPreparationHelper driverFolderPreparationHelper, REEFFileNames fileNames, [Parameter(typeof(AzureBatchAccountName))] string azureBatchAccountName, [Parameter(typeof(AzureBatchAccountUri))] string azureBatchAccountUri, [Parameter(typeof(AzureBatchPoolId))] string azureBatchPoolId, [Parameter(typeof(AzureStorageAccountName))] string azureStorageAccountName, [Parameter(typeof(AzureStorageContainerName))] string azureStorageContainerName, [Parameter(typeof(AzureBatchPoolDriverPortsList))] List <string> azureBatchPoolDriverPortsList, [Parameter(typeof(ContainerRegistryServer))] string containerRegistryServer, [Parameter(typeof(ContainerRegistryUsername))] string containerRegistryUsername, [Parameter(typeof(ContainerRegistryPassword))] string containerRegistryPassword, [Parameter(typeof(ContainerImageName))] string containerImageName) { _resourceArchiveFileGenerator = resourceArchiveFileGenerator; _driverFolderPreparationHelper = driverFolderPreparationHelper; _fileNames = fileNames; _avroAzureBatchJobSubmissionParameters = new AvroAzureBatchJobSubmissionParameters { AzureBatchAccountName = azureBatchAccountName, AzureBatchAccountUri = azureBatchAccountUri, AzureBatchPoolId = azureBatchPoolId, AzureStorageAccountName = azureStorageAccountName, AzureStorageContainerName = azureStorageContainerName, AzureBatchPoolDriverPortsList = azureBatchPoolDriverPortsList, ContainerRegistryServer = containerRegistryServer, ContainerRegistryUsername = containerRegistryUsername, ContainerRegistryPassword = containerRegistryPassword, ContainerImageName = containerImageName, }; }
private ClientLauncher( ClientService clientService, JavaClientLauncher javaClientLauncher, REEFFileNames reefFileNames, IConfigurationSerializer configurationSerializer, DriverClientParameters driverRuntimeProto, IRuntimeProtoProvider runtimeProtoProvider) { _clientService = clientService; _javaClientLauncher = javaClientLauncher; _reefFileNames = reefFileNames; _configurationSerializer = configurationSerializer; _driverClientConfiguration = driverRuntimeProto.Proto; runtimeProtoProvider.SetParameters(_driverClientConfiguration); _grpcServer = new Server { Services = { BridgeClient.BindService(clientService) }, Ports = { new ServerPort("localhost", 0, ServerCredentials.Insecure) } }; _grpcServer.Start(); Log.Log(Level.Info, "Server port any {0}", _grpcServer.Ports.Any()); foreach (var serverPort in _grpcServer.Ports) { Log.Log(Level.Info, "Server port {0}", serverPort.BoundPort); _grpcServerPort = serverPort.BoundPort; } }
JobJarMaker( IResourceArchiveFileGenerator resourceArchiveFileGenerator, DriverFolderPreparationHelper driverFolderPreparationHelper, REEFFileNames fileNames, [Parameter(typeof(AzureBatchAccountKey))] string azureBatchAccountKey, [Parameter(typeof(AzureBatchAccountName))] string azureBatchAccountName, [Parameter(typeof(AzureBatchAccountUri))] string azureBatchAccountUri, [Parameter(typeof(AzureBatchPoolId))] string azureBatchPoolId, [Parameter(typeof(AzureStorageAccountKey))] string azureStorageAccountKey, [Parameter(typeof(AzureStorageAccountName))] string azureStorageAccountName, [Parameter(typeof(AzureStorageContainerName))] string azureStorageContainerName) { _resourceArchiveFileGenerator = resourceArchiveFileGenerator; _driverFolderPreparationHelper = driverFolderPreparationHelper; _fileNames = fileNames; _avroAzureBatchJobSubmissionParameters = new AvroAzureBatchJobSubmissionParameters { AzureBatchAccountKey = azureBatchAccountKey, AzureBatchAccountName = azureBatchAccountName, AzureBatchAccountUri = azureBatchAccountUri, AzureBatchPoolId = azureBatchPoolId, AzureStorageAccountKey = azureStorageAccountKey, AzureStorageAccountName = azureStorageAccountName, AzureStorageContainerName = azureStorageContainerName, AzureBatchIsWindows = true }; }
private AzureBatchDotNetClient( IInjector injector, IResourceArchiveFileGenerator resourceArchiveFileGenerator, DriverFolderPreparationHelper driverFolderPreparationHelper, AzureStorageClient azureStorageClient, REEFFileNames fileNames, AzureBatchFileNames azbatchFileNames, JobRequestBuilderFactory jobRequestBuilderFactory, AzureBatchService batchService, JobJarMaker jobJarMaker, //// Those parameters are used in AzureBatchJobSubmissionResult, but could not be injected there. //// It introduces circular injection issues, as all classes constructor inherited from JobSubmissionResult has reference to IREEFClient. //// TODO: [REEF-2020] Refactor IJobSubmissionResult Interface and JobSubmissionResult implementation [Parameter(typeof(DriverHTTPConnectionRetryInterval))] int retryInterval, [Parameter(typeof(DriverHTTPConnectionAttempts))] int numberOfRetries) { _injector = injector; _fileNames = fileNames; _azbatchFileNames = azbatchFileNames; _driverFolderPreparationHelper = driverFolderPreparationHelper; _azureStorageClient = azureStorageClient; _jobRequestBuilderFactory = jobRequestBuilderFactory; _batchService = batchService; _jobJarMaker = jobJarMaker; _retryInterval = retryInterval; _numberOfRetries = numberOfRetries; }
private NetworkTransport( ILocalAddressProvider localAddressProvider, ProtocolSerializer serializer, LocalObserver localObserver, IRemoteManagerFactory remoteManagerFactory, REEFFileNames fileNames) { _serializer = serializer; _fileNames = fileNames; // Instantiate the remote manager. _remoteManager = remoteManagerFactory.GetInstance(localAddressProvider.LocalAddress, new ByteCodec()); // Listen to the java bridge on the local end point. _remoteManager.RegisterObserver(localObserver); Logger.Log(Level.Info, "Local observer listening to java bridge on: [{0}]", _remoteManager.LocalEndpoint); // Instantiate a remote observer to send messages to the java bridge. IPEndPoint javaIpEndPoint = GetJavaBridgeEndpoint(); Logger.Log(Level.Info, "Connecting to java bridge on: [{0}]", javaIpEndPoint); _remoteObserver = _remoteManager.GetRemoteObserver(javaIpEndPoint); // Negotiate the protocol. Send(0, new BridgeProtocol(100)); }
private JobSubmissionDirectoryProvider( [Parameter(typeof(JobSubmissionDirectoryPrefixParameter))] string jobSubmissionDirectoryPrefix, REEFFileNames fileNames) { _fileNames = fileNames; _jobSubmissionDirectoryPrefix = jobSubmissionDirectoryPrefix; }
internal DriverFolderPreparationHelper( REEFFileNames fileNames, AvroConfigurationSerializer configurationSerializer, FileSets fileSets) { _fileNames = fileNames; _configurationSerializer = configurationSerializer; _fileSets = fileSets; }
private LocalClient( DriverFolderPreparationHelper driverFolderPreparationHelper, [Parameter(typeof(NumberOfEvaluators))] int numberOfEvaluators, IJavaClientLauncher javaClientLauncher, REEFFileNames fileNames) : this(driverFolderPreparationHelper, Path.GetTempPath(), numberOfEvaluators, javaClientLauncher, fileNames) { // Intentionally left blank. }
private SecurityTokenWriter(REEFFileNames reefFileNames, [Parameter(typeof(SecurityTokenStrings))] ISet <string> serializedTokenStrings) { _securityTokensFile = reefFileNames.GetSecurityTokenFileName(); _tokens = serializedTokenStrings.Select(serializedToken => { var token = JsonConvert.DeserializeObject <SecurityToken>(serializedToken); return(new SecurityToken(token.kind, token.service, token.key, token.password)); }).ToList(); }
protected AbstractCommandBuilder( REEFFileNames fileNames, AzureBatchFileNames azureBatchFileNames, string commandPrefix, string osCommandFormat) { _fileNames = fileNames; _osCommandFormat = osCommandFormat; _commandPrefix = commandPrefix; _azureBatchFileNames = azureBatchFileNames; }
internal DriverFolderPreparationHelper( REEFFileNames fileNames, AvroConfigurationSerializer configurationSerializer, FileSets fileSets, [Parameter(typeof(EnvironmentDriverConfigurationProviders))] ISet <IConfigurationProvider> driverConfigurationProviders) { _fileNames = fileNames; _configurationSerializer = configurationSerializer; _fileSets = fileSets; _driverConfigurationProviders = driverConfigurationProviders; }
private YarnREEFParamSerializer( REEFFileNames fileNames, [Parameter(typeof(SecurityTokenKindParameter))] string securityTokenKind, [Parameter(typeof(SecurityTokenServiceParameter))] string securityTokenService, [Parameter(typeof(JobSubmissionDirectoryPrefixParameter))] string jobSubmissionPrefix) { _fileNames = fileNames; _jobSubmissionPrefix = jobSubmissionPrefix; _securityTokenKind = securityTokenKind; _securityTokenService = securityTokenService; }
private LocalClient(DriverFolderPreparationHelper driverFolderPreparationHelper, [Parameter(typeof(LocalRuntimeDirectory))] string runtimeFolder, [Parameter(typeof(NumberOfEvaluators))] int maxNumberOfConcurrentEvaluators, IJavaClientLauncher javaClientLauncher, REEFFileNames fileNames) { _driverFolderPreparationHelper = driverFolderPreparationHelper; _runtimeFolder = runtimeFolder; _maxNumberOfConcurrentEvaluators = maxNumberOfConcurrentEvaluators; _javaClientLauncher = javaClientLauncher; _fileNames = fileNames; }
private WindowsYarnJobCommandProvider( [Parameter(typeof(EnableDebugLogging))] bool enableDebugLogging, [Parameter(typeof(DriverMaxMemoryAllicationPoolSizeMB))] int driverMaxMemoryAllocationPoolSizeMB, [Parameter(typeof(DriverMaxPermSizeMB))] int driverMaxPermSizeMB, IYarnCommandLineEnvironment yarnCommandLineEnvironment, REEFFileNames fileNames) { _yarnCommandLineEnvironment = yarnCommandLineEnvironment; _enableDebugLogging = enableDebugLogging; _fileNames = fileNames; _driverMaxMemoryAllocationPoolSizeMB = driverMaxMemoryAllocationPoolSizeMB; _driverMaxPermSizeMB = driverMaxPermSizeMB; }
private LegacyJobResourceUploader( IJavaClientLauncher javaLauncher, IResourceArchiveFileGenerator resourceArchiveFileGenerator, IFile file, IYarnCommandLineEnvironment yarn, REEFFileNames reefFileNames) { _file = file; _resourceArchiveFileGenerator = resourceArchiveFileGenerator; _javaLauncher = javaLauncher; _javaLauncher.AddToClassPath(yarn.GetYarnClasspathList()); _reefFileNames = reefFileNames; }
private FileSystemJobResourceUploader( IResourceArchiveFileGenerator resourceArchiveFileGenerator, IFileSystem fileSystem, REEFFileNames reefFileNames, IFile file, IResourceFileRemoteUrlToClusterUrlConverter urlConverter) { _urlConverter = urlConverter; _reefFileNames = reefFileNames; _fileSystem = fileSystem; _resourceArchiveFileGenerator = resourceArchiveFileGenerator; _file = file; }
private LocalClient(DriverFolderPreparationHelper driverFolderPreparationHelper, [Parameter(typeof(LocalRuntimeDirectory))] string runtimeFolder, [Parameter(typeof(NumberOfEvaluators))] int maxNumberOfConcurrentEvaluators, IJavaClientLauncher javaClientLauncher, REEFFileNames fileNames) { _driverFolderPreparationHelper = driverFolderPreparationHelper; _runtimeFolder = runtimeFolder; _maxNumberOfConcurrentEvaluators = maxNumberOfConcurrentEvaluators; _javaClientLauncher = javaClientLauncher; _fileNames = fileNames; _localConfigurationOnDriver = TangFactory.GetTang().NewConfigurationBuilder().BindImplementation(GenericType <ILocalAddressProvider> .Class, GenericType <LoopbackLocalAddressProvider> .Class).Build(); }
private LocalClient( DriverFolderPreparationHelper driverFolderPreparationHelper, [Parameter(typeof(NumberOfEvaluators))] int numberOfEvaluators, [Parameter(typeof(DriverHTTPConnectionRetryInterval))] int retryInterval, [Parameter(typeof(DriverHTTPConnectionAttempts))] int numberOfRetries, IJavaClientLauncher javaClientLauncher, REEFFileNames fileNames, JobRequestBuilderFactory jobRequestBuilderFactory) : this(driverFolderPreparationHelper, Path.GetTempPath(), numberOfEvaluators, retryInterval, numberOfRetries, javaClientLauncher, fileNames, jobRequestBuilderFactory) { // Intentionally left blank. }
internal YarnREEFClient(IJavaClientLauncher javaClientLauncher, DriverFolderPreparationHelper driverFolderPreparationHelper, REEFFileNames fileNames, YarnCommandLineEnvironment yarn, IYarnRMClient yarnClient, YarnREEFParamSerializer paramSerializer) { _javaClientLauncher = javaClientLauncher; _javaClientLauncher.AddToClassPath(yarn.GetYarnClasspathList()); _driverFolderPreparationHelper = driverFolderPreparationHelper; _fileNames = fileNames; _yarnClient = yarnClient; _paramSerializer = paramSerializer; }
private YarnREEFDotNetClient( IInjector injector, IYarnRMClient yarnRMClient, DriverFolderPreparationHelper driverFolderPreparationHelper, IJobResourceUploader jobResourceUploader, REEFFileNames fileNames, IJobSubmissionDirectoryProvider jobSubmissionDirectoryProvider, YarnREEFDotNetParamSerializer paramSerializer) { _injector = injector; _jobSubmissionDirectoryProvider = jobSubmissionDirectoryProvider; _fileNames = fileNames; _jobResourceUploader = jobResourceUploader; _driverFolderPreparationHelper = driverFolderPreparationHelper; _yarnRMClient = yarnRMClient; _paramSerializer = paramSerializer; }
internal YarnREEFClient(IJavaClientLauncher javaClientLauncher, DriverFolderPreparationHelper driverFolderPreparationHelper, REEFFileNames fileNames, YarnCommandLineEnvironment yarn, IYarnRMClient yarnClient, [Parameter(typeof(SecurityTokenKindParameter))] string securityTokenKind, [Parameter(typeof(SecurityTokenServiceParameter))] string securityTokenService, [Parameter(typeof(JobSubmissionDirectoryPrefixParameter))] string jobSubmissionPrefix) { _jobSubmissionPrefix = jobSubmissionPrefix; _securityTokenKind = securityTokenKind; _securityTokenService = securityTokenService; _javaClientLauncher = javaClientLauncher; _javaClientLauncher.AddToClassPath(yarn.GetYarnClasspathList()); _driverFolderPreparationHelper = driverFolderPreparationHelper; _fileNames = fileNames; _yarnClient = yarnClient; }
private AzureBatchDotNetClient( IInjector injector, IResourceArchiveFileGenerator resourceArchiveFileGenerator, DriverFolderPreparationHelper driverFolderPreparationHelper, AzureStorageClient azureStorageClient, REEFFileNames fileNames, AzureBatchFileNames azbatchFileNames, JobRequestBuilderFactory jobRequestBuilderFactory, AzureBatchService batchService, JobJarMaker jobJarMaker) { _injector = injector; _fileNames = fileNames; _azbatchFileNames = azbatchFileNames; _driverFolderPreparationHelper = driverFolderPreparationHelper; _azureStorageClient = azureStorageClient; _jobRequestBuilderFactory = jobRequestBuilderFactory; _batchService = batchService; _jobJarMaker = jobJarMaker; }
internal YarnREEFClient(IJavaClientLauncher javaClientLauncher, DriverFolderPreparationHelper driverFolderPreparationHelper, REEFFileNames fileNames, YarnCommandLineEnvironment yarn, IYarnRMClient yarnClient, YarnREEFParamSerializer paramSerializer, JobRequestBuilderFactory jobRequestBuilderFactory, [Parameter(typeof(DriverHTTPConnectionRetryInterval))] int retryInterval, [Parameter(typeof(DriverHTTPConnectionAttempts))] int numberOfRetries) { _javaClientLauncher = javaClientLauncher; _javaClientLauncher.AddToClassPath(yarn.GetYarnClasspathList()); _driverFolderPreparationHelper = driverFolderPreparationHelper; _fileNames = fileNames; _yarnClient = yarnClient; _paramSerializer = paramSerializer; _jobRequestBuilderFactory = jobRequestBuilderFactory; _retryInterval = retryInterval; _numberOfRetries = numberOfRetries; }
private LocalClient(DriverFolderPreparationHelper driverFolderPreparationHelper, [Parameter(typeof(LocalRuntimeDirectory))] string runtimeFolder, [Parameter(typeof(NumberOfEvaluators))] int maxNumberOfConcurrentEvaluators, [Parameter(typeof(DriverHTTPConnectionRetryInterval))] int retryInterval, [Parameter(typeof(DriverHTTPConnectionAttempts))] int numberOfRetries, IJavaClientLauncher javaClientLauncher, REEFFileNames fileNames, JobRequestBuilderFactory jobRequestBuilderFactory) { _driverFolderPreparationHelper = driverFolderPreparationHelper; _runtimeFolder = runtimeFolder; _maxNumberOfConcurrentEvaluators = maxNumberOfConcurrentEvaluators; _retryInterval = retryInterval; _numberOfRetries = numberOfRetries; _javaClientLauncher = javaClientLauncher; _fileNames = fileNames; _jobRequestBuilderFactory = jobRequestBuilderFactory; _localConfigurationOnDriver = TangFactory.GetTang().NewConfigurationBuilder() .BindImplementation <ILocalAddressProvider, LoopbackLocalAddressProvider>() .Build(); }
internal BridgeConfigurationProvider(REEFFileNames fileNames) { _fileNames = fileNames; }
private YarnREEFDotNetParamSerializer(REEFFileNames fileNames, [Parameter(typeof(FileSystemUrl))] string fileSystemUrl) { _fileNames = fileNames; _fileSystemUrl = fileSystemUrl; }
private AzureBatchFileNames(REEFFileNames reefFileNames) { _reefFileNames = reefFileNames; }
private ClassHierarchyGeneratingDriverStartObserver(REEFFileNames fileNames) { _fileNames = fileNames; }
private WindowsCommandBuilder( REEFFileNames fileNames, AzureBatchFileNames azureBatchFileNames) : base(fileNames, azureBatchFileNames, CommandPrefix, OsCommandFormat) { }
internal FileSets(REEFFileNames fileNames) { _fileNames = fileNames; }