private JobJarMaker(
IResourceArchiveFileGenerator resourceArchiveFileGenerator,
DriverFolderPreparationHelper driverFolderPreparationHelper,
REEFFileNames fileNames,
[Parameter(typeof(AzureBatchAccountName))] string azureBatchAccountName,
[Parameter(typeof(AzureBatchAccountUri))] string azureBatchAccountUri,
[Parameter(typeof(AzureBatchPoolId))] string azureBatchPoolId,
[Parameter(typeof(AzureStorageAccountName))] string azureStorageAccountName,
[Parameter(typeof(AzureStorageContainerName))] string azureStorageContainerName,
[Parameter(typeof(AzureBatchPoolDriverPortsList))] List <string> azureBatchPoolDriverPortsList,
[Parameter(typeof(ContainerRegistryServer))] string containerRegistryServer,
[Parameter(typeof(ContainerRegistryUsername))] string containerRegistryUsername,
[Parameter(typeof(ContainerRegistryPassword))] string containerRegistryPassword,
[Parameter(typeof(ContainerImageName))] string containerImageName)
{
_resourceArchiveFileGenerator = resourceArchiveFileGenerator;
_driverFolderPreparationHelper = driverFolderPreparationHelper;
_fileNames = fileNames;
_avroAzureBatchJobSubmissionParameters = new AvroAzureBatchJobSubmissionParameters
{
AzureBatchAccountName = azureBatchAccountName,
AzureBatchAccountUri = azureBatchAccountUri,
AzureBatchPoolId = azureBatchPoolId,
AzureStorageAccountName = azureStorageAccountName,
AzureStorageContainerName = azureStorageContainerName,
AzureBatchPoolDriverPortsList = azureBatchPoolDriverPortsList,
ContainerRegistryServer = containerRegistryServer,
ContainerRegistryUsername = containerRegistryUsername,
ContainerRegistryPassword = containerRegistryPassword,
ContainerImageName = containerImageName,
};
}
private ClientLauncher(
ClientService clientService,
JavaClientLauncher javaClientLauncher,
REEFFileNames reefFileNames,
IConfigurationSerializer configurationSerializer,
DriverClientParameters driverRuntimeProto,
IRuntimeProtoProvider runtimeProtoProvider)
{
_clientService = clientService;
_javaClientLauncher = javaClientLauncher;
_reefFileNames = reefFileNames;
_configurationSerializer = configurationSerializer;
_driverClientConfiguration = driverRuntimeProto.Proto;
runtimeProtoProvider.SetParameters(_driverClientConfiguration);
_grpcServer = new Server
{
Services = { BridgeClient.BindService(clientService) },
Ports = { new ServerPort("localhost", 0, ServerCredentials.Insecure) }
};
_grpcServer.Start();
Log.Log(Level.Info, "Server port any {0}", _grpcServer.Ports.Any());
foreach (var serverPort in _grpcServer.Ports)
{
Log.Log(Level.Info, "Server port {0}", serverPort.BoundPort);
_grpcServerPort = serverPort.BoundPort;
}
}
JobJarMaker(
IResourceArchiveFileGenerator resourceArchiveFileGenerator,
DriverFolderPreparationHelper driverFolderPreparationHelper,
REEFFileNames fileNames,
[Parameter(typeof(AzureBatchAccountKey))] string azureBatchAccountKey,
[Parameter(typeof(AzureBatchAccountName))] string azureBatchAccountName,
[Parameter(typeof(AzureBatchAccountUri))] string azureBatchAccountUri,
[Parameter(typeof(AzureBatchPoolId))] string azureBatchPoolId,
[Parameter(typeof(AzureStorageAccountKey))] string azureStorageAccountKey,
[Parameter(typeof(AzureStorageAccountName))] string azureStorageAccountName,
[Parameter(typeof(AzureStorageContainerName))] string azureStorageContainerName)
{
_resourceArchiveFileGenerator = resourceArchiveFileGenerator;
_driverFolderPreparationHelper = driverFolderPreparationHelper;
_fileNames = fileNames;
_avroAzureBatchJobSubmissionParameters = new AvroAzureBatchJobSubmissionParameters
{
AzureBatchAccountKey = azureBatchAccountKey,
AzureBatchAccountName = azureBatchAccountName,
AzureBatchAccountUri = azureBatchAccountUri,
AzureBatchPoolId = azureBatchPoolId,
AzureStorageAccountKey = azureStorageAccountKey,
AzureStorageAccountName = azureStorageAccountName,
AzureStorageContainerName = azureStorageContainerName,
AzureBatchIsWindows = true
};
}
private AzureBatchDotNetClient(
IInjector injector,
IResourceArchiveFileGenerator resourceArchiveFileGenerator,
DriverFolderPreparationHelper driverFolderPreparationHelper,
AzureStorageClient azureStorageClient,
REEFFileNames fileNames,
AzureBatchFileNames azbatchFileNames,
JobRequestBuilderFactory jobRequestBuilderFactory,
AzureBatchService batchService,
JobJarMaker jobJarMaker,
//// Those parameters are used in AzureBatchJobSubmissionResult, but could not be injected there.
//// It introduces circular injection issues, as all classes constructor inherited from JobSubmissionResult has reference to IREEFClient.
//// TODO: [REEF-2020] Refactor IJobSubmissionResult Interface and JobSubmissionResult implementation
[Parameter(typeof(DriverHTTPConnectionRetryInterval))] int retryInterval,
[Parameter(typeof(DriverHTTPConnectionAttempts))] int numberOfRetries)
{
_injector = injector;
_fileNames = fileNames;
_azbatchFileNames = azbatchFileNames;
_driverFolderPreparationHelper = driverFolderPreparationHelper;
_azureStorageClient = azureStorageClient;
_jobRequestBuilderFactory = jobRequestBuilderFactory;
_batchService = batchService;
_jobJarMaker = jobJarMaker;
_retryInterval = retryInterval;
_numberOfRetries = numberOfRetries;
}
private NetworkTransport(
ILocalAddressProvider localAddressProvider,
ProtocolSerializer serializer,
LocalObserver localObserver,
IRemoteManagerFactory remoteManagerFactory,
REEFFileNames fileNames)
{
_serializer = serializer;
_fileNames = fileNames;
// Instantiate the remote manager.
_remoteManager = remoteManagerFactory.GetInstance(localAddressProvider.LocalAddress, new ByteCodec());
// Listen to the java bridge on the local end point.
_remoteManager.RegisterObserver(localObserver);
Logger.Log(Level.Info, "Local observer listening to java bridge on: [{0}]", _remoteManager.LocalEndpoint);
// Instantiate a remote observer to send messages to the java bridge.
IPEndPoint javaIpEndPoint = GetJavaBridgeEndpoint();
Logger.Log(Level.Info, "Connecting to java bridge on: [{0}]", javaIpEndPoint);
_remoteObserver = _remoteManager.GetRemoteObserver(javaIpEndPoint);
// Negotiate the protocol.
Send(0, new BridgeProtocol(100));
}
private JobSubmissionDirectoryProvider(
[Parameter(typeof(JobSubmissionDirectoryPrefixParameter))]
string jobSubmissionDirectoryPrefix,
REEFFileNames fileNames)
{
_fileNames = fileNames;
_jobSubmissionDirectoryPrefix = jobSubmissionDirectoryPrefix;
}
internal DriverFolderPreparationHelper(
REEFFileNames fileNames,
AvroConfigurationSerializer configurationSerializer,
FileSets fileSets)
{
_fileNames = fileNames;
_configurationSerializer = configurationSerializer;
_fileSets = fileSets;
}
private LocalClient(
DriverFolderPreparationHelper driverFolderPreparationHelper,
[Parameter(typeof(NumberOfEvaluators))] int numberOfEvaluators,
IJavaClientLauncher javaClientLauncher,
REEFFileNames fileNames)
: this(driverFolderPreparationHelper, Path.GetTempPath(), numberOfEvaluators, javaClientLauncher, fileNames)
{
// Intentionally left blank.
}
private SecurityTokenWriter(REEFFileNames reefFileNames,
[Parameter(typeof(SecurityTokenStrings))] ISet <string> serializedTokenStrings)
{
_securityTokensFile = reefFileNames.GetSecurityTokenFileName();
_tokens = serializedTokenStrings.Select(serializedToken =>
{
var token = JsonConvert.DeserializeObject <SecurityToken>(serializedToken);
return(new SecurityToken(token.kind, token.service, token.key, token.password));
}).ToList();
}
protected AbstractCommandBuilder(
REEFFileNames fileNames,
AzureBatchFileNames azureBatchFileNames,
string commandPrefix,
string osCommandFormat)
{
_fileNames = fileNames;
_osCommandFormat = osCommandFormat;
_commandPrefix = commandPrefix;
_azureBatchFileNames = azureBatchFileNames;
}
internal DriverFolderPreparationHelper(
REEFFileNames fileNames,
AvroConfigurationSerializer configurationSerializer,
FileSets fileSets,
[Parameter(typeof(EnvironmentDriverConfigurationProviders))] ISet <IConfigurationProvider> driverConfigurationProviders)
{
_fileNames = fileNames;
_configurationSerializer = configurationSerializer;
_fileSets = fileSets;
_driverConfigurationProviders = driverConfigurationProviders;
}
private YarnREEFParamSerializer(
REEFFileNames fileNames,
[Parameter(typeof(SecurityTokenKindParameter))] string securityTokenKind,
[Parameter(typeof(SecurityTokenServiceParameter))] string securityTokenService,
[Parameter(typeof(JobSubmissionDirectoryPrefixParameter))] string jobSubmissionPrefix)
{
_fileNames = fileNames;
_jobSubmissionPrefix = jobSubmissionPrefix;
_securityTokenKind = securityTokenKind;
_securityTokenService = securityTokenService;
}
private LocalClient(DriverFolderPreparationHelper driverFolderPreparationHelper,
[Parameter(typeof(LocalRuntimeDirectory))] string runtimeFolder,
[Parameter(typeof(NumberOfEvaluators))] int maxNumberOfConcurrentEvaluators,
IJavaClientLauncher javaClientLauncher,
REEFFileNames fileNames)
{
_driverFolderPreparationHelper = driverFolderPreparationHelper;
_runtimeFolder = runtimeFolder;
_maxNumberOfConcurrentEvaluators = maxNumberOfConcurrentEvaluators;
_javaClientLauncher = javaClientLauncher;
_fileNames = fileNames;
}
private WindowsYarnJobCommandProvider(
[Parameter(typeof(EnableDebugLogging))] bool enableDebugLogging,
[Parameter(typeof(DriverMaxMemoryAllicationPoolSizeMB))] int driverMaxMemoryAllocationPoolSizeMB,
[Parameter(typeof(DriverMaxPermSizeMB))] int driverMaxPermSizeMB,
IYarnCommandLineEnvironment yarnCommandLineEnvironment,
REEFFileNames fileNames)
{
_yarnCommandLineEnvironment = yarnCommandLineEnvironment;
_enableDebugLogging = enableDebugLogging;
_fileNames = fileNames;
_driverMaxMemoryAllocationPoolSizeMB = driverMaxMemoryAllocationPoolSizeMB;
_driverMaxPermSizeMB = driverMaxPermSizeMB;
}
private LegacyJobResourceUploader(
IJavaClientLauncher javaLauncher,
IResourceArchiveFileGenerator resourceArchiveFileGenerator,
IFile file,
IYarnCommandLineEnvironment yarn,
REEFFileNames reefFileNames)
{
_file = file;
_resourceArchiveFileGenerator = resourceArchiveFileGenerator;
_javaLauncher = javaLauncher;
_javaLauncher.AddToClassPath(yarn.GetYarnClasspathList());
_reefFileNames = reefFileNames;
}
private FileSystemJobResourceUploader(
IResourceArchiveFileGenerator resourceArchiveFileGenerator,
IFileSystem fileSystem,
REEFFileNames reefFileNames,
IFile file,
IResourceFileRemoteUrlToClusterUrlConverter urlConverter)
{
_urlConverter = urlConverter;
_reefFileNames = reefFileNames;
_fileSystem = fileSystem;
_resourceArchiveFileGenerator = resourceArchiveFileGenerator;
_file = file;
}
private LocalClient(DriverFolderPreparationHelper driverFolderPreparationHelper,
[Parameter(typeof(LocalRuntimeDirectory))] string runtimeFolder,
[Parameter(typeof(NumberOfEvaluators))] int maxNumberOfConcurrentEvaluators,
IJavaClientLauncher javaClientLauncher,
REEFFileNames fileNames)
{
_driverFolderPreparationHelper = driverFolderPreparationHelper;
_runtimeFolder = runtimeFolder;
_maxNumberOfConcurrentEvaluators = maxNumberOfConcurrentEvaluators;
_javaClientLauncher = javaClientLauncher;
_fileNames = fileNames;
_localConfigurationOnDriver = TangFactory.GetTang().NewConfigurationBuilder().BindImplementation(GenericType <ILocalAddressProvider> .Class, GenericType <LoopbackLocalAddressProvider> .Class).Build();
}
private LocalClient(
DriverFolderPreparationHelper driverFolderPreparationHelper,
[Parameter(typeof(NumberOfEvaluators))] int numberOfEvaluators,
[Parameter(typeof(DriverHTTPConnectionRetryInterval))] int retryInterval,
[Parameter(typeof(DriverHTTPConnectionAttempts))] int numberOfRetries,
IJavaClientLauncher javaClientLauncher,
REEFFileNames fileNames,
JobRequestBuilderFactory jobRequestBuilderFactory)
: this(driverFolderPreparationHelper, Path.GetTempPath(),
numberOfEvaluators, retryInterval, numberOfRetries, javaClientLauncher, fileNames, jobRequestBuilderFactory)
{
// Intentionally left blank.
}
internal YarnREEFClient(IJavaClientLauncher javaClientLauncher,
DriverFolderPreparationHelper driverFolderPreparationHelper,
REEFFileNames fileNames,
YarnCommandLineEnvironment yarn,
IYarnRMClient yarnClient,
YarnREEFParamSerializer paramSerializer)
{
_javaClientLauncher = javaClientLauncher;
_javaClientLauncher.AddToClassPath(yarn.GetYarnClasspathList());
_driverFolderPreparationHelper = driverFolderPreparationHelper;
_fileNames = fileNames;
_yarnClient = yarnClient;
_paramSerializer = paramSerializer;
}
private YarnREEFDotNetClient(
IInjector injector,
IYarnRMClient yarnRMClient,
DriverFolderPreparationHelper driverFolderPreparationHelper,
IJobResourceUploader jobResourceUploader,
REEFFileNames fileNames,
IJobSubmissionDirectoryProvider jobSubmissionDirectoryProvider,
YarnREEFDotNetParamSerializer paramSerializer)
{
_injector = injector;
_jobSubmissionDirectoryProvider = jobSubmissionDirectoryProvider;
_fileNames = fileNames;
_jobResourceUploader = jobResourceUploader;
_driverFolderPreparationHelper = driverFolderPreparationHelper;
_yarnRMClient = yarnRMClient;
_paramSerializer = paramSerializer;
}
internal YarnREEFClient(IJavaClientLauncher javaClientLauncher,
DriverFolderPreparationHelper driverFolderPreparationHelper,
REEFFileNames fileNames,
YarnCommandLineEnvironment yarn,
IYarnRMClient yarnClient,
[Parameter(typeof(SecurityTokenKindParameter))] string securityTokenKind,
[Parameter(typeof(SecurityTokenServiceParameter))] string securityTokenService,
[Parameter(typeof(JobSubmissionDirectoryPrefixParameter))] string jobSubmissionPrefix)
{
_jobSubmissionPrefix = jobSubmissionPrefix;
_securityTokenKind = securityTokenKind;
_securityTokenService = securityTokenService;
_javaClientLauncher = javaClientLauncher;
_javaClientLauncher.AddToClassPath(yarn.GetYarnClasspathList());
_driverFolderPreparationHelper = driverFolderPreparationHelper;
_fileNames = fileNames;
_yarnClient = yarnClient;
}
private AzureBatchDotNetClient(
IInjector injector,
IResourceArchiveFileGenerator resourceArchiveFileGenerator,
DriverFolderPreparationHelper driverFolderPreparationHelper,
AzureStorageClient azureStorageClient,
REEFFileNames fileNames,
AzureBatchFileNames azbatchFileNames,
JobRequestBuilderFactory jobRequestBuilderFactory,
AzureBatchService batchService,
JobJarMaker jobJarMaker)
{
_injector = injector;
_fileNames = fileNames;
_azbatchFileNames = azbatchFileNames;
_driverFolderPreparationHelper = driverFolderPreparationHelper;
_azureStorageClient = azureStorageClient;
_jobRequestBuilderFactory = jobRequestBuilderFactory;
_batchService = batchService;
_jobJarMaker = jobJarMaker;
}
internal YarnREEFClient(IJavaClientLauncher javaClientLauncher,
DriverFolderPreparationHelper driverFolderPreparationHelper,
REEFFileNames fileNames,
YarnCommandLineEnvironment yarn,
IYarnRMClient yarnClient,
YarnREEFParamSerializer paramSerializer,
JobRequestBuilderFactory jobRequestBuilderFactory,
[Parameter(typeof(DriverHTTPConnectionRetryInterval))] int retryInterval,
[Parameter(typeof(DriverHTTPConnectionAttempts))] int numberOfRetries)
{
_javaClientLauncher = javaClientLauncher;
_javaClientLauncher.AddToClassPath(yarn.GetYarnClasspathList());
_driverFolderPreparationHelper = driverFolderPreparationHelper;
_fileNames = fileNames;
_yarnClient = yarnClient;
_paramSerializer = paramSerializer;
_jobRequestBuilderFactory = jobRequestBuilderFactory;
_retryInterval = retryInterval;
_numberOfRetries = numberOfRetries;
}
private LocalClient(DriverFolderPreparationHelper driverFolderPreparationHelper,
[Parameter(typeof(LocalRuntimeDirectory))] string runtimeFolder,
[Parameter(typeof(NumberOfEvaluators))] int maxNumberOfConcurrentEvaluators,
[Parameter(typeof(DriverHTTPConnectionRetryInterval))] int retryInterval,
[Parameter(typeof(DriverHTTPConnectionAttempts))] int numberOfRetries,
IJavaClientLauncher javaClientLauncher,
REEFFileNames fileNames,
JobRequestBuilderFactory jobRequestBuilderFactory)
{
_driverFolderPreparationHelper = driverFolderPreparationHelper;
_runtimeFolder = runtimeFolder;
_maxNumberOfConcurrentEvaluators = maxNumberOfConcurrentEvaluators;
_retryInterval = retryInterval;
_numberOfRetries = numberOfRetries;
_javaClientLauncher = javaClientLauncher;
_fileNames = fileNames;
_jobRequestBuilderFactory = jobRequestBuilderFactory;
_localConfigurationOnDriver = TangFactory.GetTang().NewConfigurationBuilder()
.BindImplementation <ILocalAddressProvider, LoopbackLocalAddressProvider>()
.Build();
}
internal BridgeConfigurationProvider(REEFFileNames fileNames)
{
_fileNames = fileNames;
}
private YarnREEFDotNetParamSerializer(REEFFileNames fileNames,
[Parameter(typeof(FileSystemUrl))] string fileSystemUrl)
{
_fileNames = fileNames;
_fileSystemUrl = fileSystemUrl;
}
private AzureBatchFileNames(REEFFileNames reefFileNames)
{
_reefFileNames = reefFileNames;
}
private ClassHierarchyGeneratingDriverStartObserver(REEFFileNames fileNames)
{
_fileNames = fileNames;
}
private WindowsCommandBuilder(
REEFFileNames fileNames,
AzureBatchFileNames azureBatchFileNames) : base(fileNames, azureBatchFileNames,
CommandPrefix, OsCommandFormat)
{
}
internal FileSets(REEFFileNames fileNames)
{
_fileNames = fileNames;
}
