JobJarMaker(
IResourceArchiveFileGenerator resourceArchiveFileGenerator,
DriverFolderPreparationHelper driverFolderPreparationHelper,
REEFFileNames fileNames,
[Parameter(typeof(AzureBatchAccountKey))] string azureBatchAccountKey,
[Parameter(typeof(AzureBatchAccountName))] string azureBatchAccountName,
[Parameter(typeof(AzureBatchAccountUri))] string azureBatchAccountUri,
[Parameter(typeof(AzureBatchPoolId))] string azureBatchPoolId,
[Parameter(typeof(AzureStorageAccountKey))] string azureStorageAccountKey,
[Parameter(typeof(AzureStorageAccountName))] string azureStorageAccountName,
[Parameter(typeof(AzureStorageContainerName))] string azureStorageContainerName)
{
_resourceArchiveFileGenerator = resourceArchiveFileGenerator;
_driverFolderPreparationHelper = driverFolderPreparationHelper;
_fileNames = fileNames;
_avroAzureBatchJobSubmissionParameters = new AvroAzureBatchJobSubmissionParameters
{
AzureBatchAccountKey = azureBatchAccountKey,
AzureBatchAccountName = azureBatchAccountName,
AzureBatchAccountUri = azureBatchAccountUri,
AzureBatchPoolId = azureBatchPoolId,
AzureStorageAccountKey = azureStorageAccountKey,
AzureStorageAccountName = azureStorageAccountName,
AzureStorageContainerName = azureStorageContainerName,
AzureBatchIsWindows = true
};
}
private JobJarMaker(
IResourceArchiveFileGenerator resourceArchiveFileGenerator,
DriverFolderPreparationHelper driverFolderPreparationHelper,
REEFFileNames fileNames,
[Parameter(typeof(AzureBatchAccountName))] string azureBatchAccountName,
[Parameter(typeof(AzureBatchAccountUri))] string azureBatchAccountUri,
[Parameter(typeof(AzureBatchPoolId))] string azureBatchPoolId,
[Parameter(typeof(AzureStorageAccountName))] string azureStorageAccountName,
[Parameter(typeof(AzureStorageContainerName))] string azureStorageContainerName,
[Parameter(typeof(AzureBatchPoolDriverPortsList))] List <string> azureBatchPoolDriverPortsList,
[Parameter(typeof(ContainerRegistryServer))] string containerRegistryServer,
[Parameter(typeof(ContainerRegistryUsername))] string containerRegistryUsername,
[Parameter(typeof(ContainerRegistryPassword))] string containerRegistryPassword,
[Parameter(typeof(ContainerImageName))] string containerImageName)
{
_resourceArchiveFileGenerator = resourceArchiveFileGenerator;
_driverFolderPreparationHelper = driverFolderPreparationHelper;
_fileNames = fileNames;
_avroAzureBatchJobSubmissionParameters = new AvroAzureBatchJobSubmissionParameters
{
AzureBatchAccountName = azureBatchAccountName,
AzureBatchAccountUri = azureBatchAccountUri,
AzureBatchPoolId = azureBatchPoolId,
AzureStorageAccountName = azureStorageAccountName,
AzureStorageContainerName = azureStorageContainerName,
AzureBatchPoolDriverPortsList = azureBatchPoolDriverPortsList,
ContainerRegistryServer = containerRegistryServer,
ContainerRegistryUsername = containerRegistryUsername,
ContainerRegistryPassword = containerRegistryPassword,
ContainerImageName = containerImageName,
};
}
private AzureBatchDotNetClient(
IInjector injector,
IResourceArchiveFileGenerator resourceArchiveFileGenerator,
DriverFolderPreparationHelper driverFolderPreparationHelper,
AzureStorageClient azureStorageClient,
REEFFileNames fileNames,
AzureBatchFileNames azbatchFileNames,
JobRequestBuilderFactory jobRequestBuilderFactory,
AzureBatchService batchService,
JobJarMaker jobJarMaker,
//// Those parameters are used in AzureBatchJobSubmissionResult, but could not be injected there.
//// It introduces circular injection issues, as all classes constructor inherited from JobSubmissionResult has reference to IREEFClient.
//// TODO: [REEF-2020] Refactor IJobSubmissionResult Interface and JobSubmissionResult implementation
[Parameter(typeof(DriverHTTPConnectionRetryInterval))] int retryInterval,
[Parameter(typeof(DriverHTTPConnectionAttempts))] int numberOfRetries)
{
_injector = injector;
_fileNames = fileNames;
_azbatchFileNames = azbatchFileNames;
_driverFolderPreparationHelper = driverFolderPreparationHelper;
_azureStorageClient = azureStorageClient;
_jobRequestBuilderFactory = jobRequestBuilderFactory;
_batchService = batchService;
_jobJarMaker = jobJarMaker;
_retryInterval = retryInterval;
_numberOfRetries = numberOfRetries;
}
private LocalClient(
DriverFolderPreparationHelper driverFolderPreparationHelper,
[Parameter(typeof(NumberOfEvaluators))] int numberOfEvaluators, JavaClientLauncher javaClientLauncher)
: this(driverFolderPreparationHelper, Path.GetTempPath(), numberOfEvaluators, javaClientLauncher)
{
// Intentionally left blank.
}
internal YARNClient(JavaClientLauncher javaClientLauncher,
DriverFolderPreparationHelper driverFolderPreparationHelper,
YarnCommandLineEnvironment yarn)
{
_javaClientLauncher = javaClientLauncher;
_javaClientLauncher.AddToClassPath(yarn.GetYarnClasspathList());
_driverFolderPreparationHelper = driverFolderPreparationHelper;
}
private LocalClient(DriverFolderPreparationHelper driverFolderPreparationHelper,
[Parameter(typeof(LocalRuntimeDirectory))] string runtimeFolder,
[Parameter(typeof(NumberOfEvaluators))] int numberOfEvaluators, JavaClientLauncher javaClientLauncher)
{
_driverFolderPreparationHelper = driverFolderPreparationHelper;
_runtimeFolder = runtimeFolder;
_numberOfEvaluators = numberOfEvaluators;
_javaClientLauncher = javaClientLauncher;
}
private LocalClient(DriverFolderPreparationHelper driverFolderPreparationHelper,
[Parameter(typeof(LocalRuntimeDirectory))] string runtimeFolder,
[Parameter(typeof(NumberOfEvaluators))] int maxNumberOfConcurrentEvaluators,
IJavaClientLauncher javaClientLauncher,
REEFFileNames fileNames)
{
_driverFolderPreparationHelper = driverFolderPreparationHelper;
_runtimeFolder = runtimeFolder;
_maxNumberOfConcurrentEvaluators = maxNumberOfConcurrentEvaluators;
_javaClientLauncher = javaClientLauncher;
_fileNames = fileNames;
}
private LocalClient(
DriverFolderPreparationHelper driverFolderPreparationHelper,
[Parameter(typeof(NumberOfEvaluators))] int numberOfEvaluators,
[Parameter(typeof(DriverHTTPConnectionRetryInterval))] int retryInterval,
[Parameter(typeof(DriverHTTPConnectionAttempts))] int numberOfRetries,
IJavaClientLauncher javaClientLauncher,
REEFFileNames fileNames,
JobRequestBuilderFactory jobRequestBuilderFactory)
: this(driverFolderPreparationHelper, Path.GetTempPath(),
numberOfEvaluators, retryInterval, numberOfRetries, javaClientLauncher, fileNames, jobRequestBuilderFactory)
{
// Intentionally left blank.
}
private LocalClient(DriverFolderPreparationHelper driverFolderPreparationHelper,
[Parameter(typeof(LocalRuntimeDirectory))] string runtimeFolder,
[Parameter(typeof(NumberOfEvaluators))] int maxNumberOfConcurrentEvaluators,
IJavaClientLauncher javaClientLauncher,
REEFFileNames fileNames)
{
_driverFolderPreparationHelper = driverFolderPreparationHelper;
_runtimeFolder = runtimeFolder;
_maxNumberOfConcurrentEvaluators = maxNumberOfConcurrentEvaluators;
_javaClientLauncher = javaClientLauncher;
_fileNames = fileNames;
_localConfigurationOnDriver = TangFactory.GetTang().NewConfigurationBuilder().BindImplementation(GenericType <ILocalAddressProvider> .Class, GenericType <LoopbackLocalAddressProvider> .Class).Build();
}
internal YarnREEFClient(IJavaClientLauncher javaClientLauncher,
DriverFolderPreparationHelper driverFolderPreparationHelper,
REEFFileNames fileNames,
YarnCommandLineEnvironment yarn,
IYarnRMClient yarnClient,
YarnREEFParamSerializer paramSerializer)
{
_javaClientLauncher = javaClientLauncher;
_javaClientLauncher.AddToClassPath(yarn.GetYarnClasspathList());
_driverFolderPreparationHelper = driverFolderPreparationHelper;
_fileNames = fileNames;
_yarnClient = yarnClient;
_paramSerializer = paramSerializer;
}
private YarnREEFDotNetClient(
IInjector injector,
IYarnRMClient yarnRMClient,
DriverFolderPreparationHelper driverFolderPreparationHelper,
IJobResourceUploader jobResourceUploader,
REEFFileNames fileNames,
IJobSubmissionDirectoryProvider jobSubmissionDirectoryProvider,
YarnREEFDotNetParamSerializer paramSerializer)
{
_injector = injector;
_jobSubmissionDirectoryProvider = jobSubmissionDirectoryProvider;
_fileNames = fileNames;
_jobResourceUploader = jobResourceUploader;
_driverFolderPreparationHelper = driverFolderPreparationHelper;
_yarnRMClient = yarnRMClient;
_paramSerializer = paramSerializer;
}
internal YarnREEFClient(IJavaClientLauncher javaClientLauncher,
DriverFolderPreparationHelper driverFolderPreparationHelper,
REEFFileNames fileNames,
YarnCommandLineEnvironment yarn,
IYarnRMClient yarnClient,
[Parameter(typeof(SecurityTokenKindParameter))] string securityTokenKind,
[Parameter(typeof(SecurityTokenServiceParameter))] string securityTokenService,
[Parameter(typeof(JobSubmissionDirectoryPrefixParameter))] string jobSubmissionPrefix)
{
_jobSubmissionPrefix = jobSubmissionPrefix;
_securityTokenKind = securityTokenKind;
_securityTokenService = securityTokenService;
_javaClientLauncher = javaClientLauncher;
_javaClientLauncher.AddToClassPath(yarn.GetYarnClasspathList());
_driverFolderPreparationHelper = driverFolderPreparationHelper;
_fileNames = fileNames;
_yarnClient = yarnClient;
}
internal YarnREEFClient(IJavaClientLauncher javaClientLauncher,
DriverFolderPreparationHelper driverFolderPreparationHelper,
REEFFileNames fileNames,
YarnCommandLineEnvironment yarn,
IYarnRMClient yarnClient,
YarnREEFParamSerializer paramSerializer,
JobRequestBuilderFactory jobRequestBuilderFactory,
[Parameter(typeof(DriverHTTPConnectionRetryInterval))] int retryInterval,
[Parameter(typeof(DriverHTTPConnectionAttempts))] int numberOfRetries)
{
_javaClientLauncher = javaClientLauncher;
_javaClientLauncher.AddToClassPath(yarn.GetYarnClasspathList());
_driverFolderPreparationHelper = driverFolderPreparationHelper;
_fileNames = fileNames;
_yarnClient = yarnClient;
_paramSerializer = paramSerializer;
_jobRequestBuilderFactory = jobRequestBuilderFactory;
_retryInterval = retryInterval;
_numberOfRetries = numberOfRetries;
}
private AzureBatchDotNetClient(
IInjector injector,
IResourceArchiveFileGenerator resourceArchiveFileGenerator,
DriverFolderPreparationHelper driverFolderPreparationHelper,
AzureStorageClient azureStorageClient,
REEFFileNames fileNames,
AzureBatchFileNames azbatchFileNames,
JobRequestBuilderFactory jobRequestBuilderFactory,
AzureBatchService batchService,
JobJarMaker jobJarMaker)
{
_injector = injector;
_fileNames = fileNames;
_azbatchFileNames = azbatchFileNames;
_driverFolderPreparationHelper = driverFolderPreparationHelper;
_azureStorageClient = azureStorageClient;
_jobRequestBuilderFactory = jobRequestBuilderFactory;
_batchService = batchService;
_jobJarMaker = jobJarMaker;
}
private LocalClient(DriverFolderPreparationHelper driverFolderPreparationHelper,
[Parameter(typeof(LocalRuntimeDirectory))] string runtimeFolder,
[Parameter(typeof(NumberOfEvaluators))] int maxNumberOfConcurrentEvaluators,
[Parameter(typeof(DriverHTTPConnectionRetryInterval))] int retryInterval,
[Parameter(typeof(DriverHTTPConnectionAttempts))] int numberOfRetries,
IJavaClientLauncher javaClientLauncher,
REEFFileNames fileNames,
JobRequestBuilderFactory jobRequestBuilderFactory)
{
_driverFolderPreparationHelper = driverFolderPreparationHelper;
_runtimeFolder = runtimeFolder;
_maxNumberOfConcurrentEvaluators = maxNumberOfConcurrentEvaluators;
_retryInterval = retryInterval;
_numberOfRetries = numberOfRetries;
_javaClientLauncher = javaClientLauncher;
_fileNames = fileNames;
_jobRequestBuilderFactory = jobRequestBuilderFactory;
_localConfigurationOnDriver = TangFactory.GetTang().NewConfigurationBuilder()
.BindImplementation <ILocalAddressProvider, LoopbackLocalAddressProvider>()
.Build();
}
