public JobResource UploadJobResource(string driverLocalFolderPath)
{
driverLocalFolderPath = driverLocalFolderPath.TrimEnd('\\') + @"\";
var driverUploadPath = _jobSubmissionDirectoryProvider.GetJobSubmissionRemoteDirectory().TrimEnd('/') + @"/";
Log.Log(Level.Verbose, "DriverFolderPath: {0} DriverUploadPath: {1}", driverLocalFolderPath, driverUploadPath);
var archivePath = _resourceArchiveFileGenerator.CreateArchiveToUpload(driverLocalFolderPath);
var destinationPath = driverUploadPath + Path.GetFileName(archivePath);
var remoteFileUri = _fileSystem.CreateUriForPath(destinationPath);
Log.Log(Level.Verbose, @"Copy {0} to {1}", archivePath, remoteFileUri);
var parentDirectoryUri = _fileSystem.CreateUriForPath(driverUploadPath);
_fileSystem.CreateDirectory(parentDirectoryUri);
_fileSystem.CopyFromLocal(archivePath, remoteFileUri);
var fileStatus = _fileSystem.GetFileStatus(remoteFileUri);
return(new JobResource
{
LastModificationUnixTimestamp = DateTimeToUnixTimestamp(fileStatus.ModificationTime),
RemoteUploadPath = remoteFileUri.AbsoluteUri,
ResourceSize = fileStatus.LengthBytes
});
}
public async Task <JobResource> UploadArchiveResourceAsync(string driverLocalFolderPath, string remoteUploadDirectoryPath)
{
driverLocalFolderPath = driverLocalFolderPath.TrimEnd('\\') + @"\";
var driverUploadPath = remoteUploadDirectoryPath.TrimEnd('/') + @"/";
Log.Log(Level.Info, "DriverFolderPath: {0} DriverUploadPath: {1}", driverLocalFolderPath, driverUploadPath);
var archivePath = _resourceArchiveFileGenerator.CreateArchiveToUpload(driverLocalFolderPath);
return(await UploadResourceAndGetInfoAsync(archivePath, ResourceType.ARCHIVE, driverUploadPath, _reefFileNames.GetReefFolderName()));
}
/// <summary>
/// Creates a JAR file for the job submission.
/// </summary>
/// <param name="jobRequest">Job request received from the client code.</param>
/// <param name="azureBatchjobId">Azure Batch job Id going to be launched.</param>
/// <returns>A string path to file.</returns>
public string CreateJobSubmissionJAR(JobRequest jobRequest, string azureBatchjobId)
{
_avroAzureBatchJobSubmissionParameters.sharedJobSubmissionParameters = new AvroJobSubmissionParameters
{
jobId = jobRequest.JobIdentifier,
//// This is dummy in Azure Batch, as it does not use jobSubmissionFolder in Azure Batch.
jobSubmissionFolder = Path.PathSeparator.ToString()
};
string localDriverFolderPath = CreateDriverFolder(azureBatchjobId);
_driverFolderPreparationHelper.PrepareDriverFolderWithGlobalBridgeJar(jobRequest.AppParameters, localDriverFolderPath);
SerializeJobFile(localDriverFolderPath, _avroAzureBatchJobSubmissionParameters);
return(_resourceArchiveFileGenerator.CreateArchiveToUpload(localDriverFolderPath));
}
public JobResource UploadJobResource(string driverLocalFolderPath)
{
driverLocalFolderPath = driverLocalFolderPath.TrimEnd('\\') + @"\";
string driverUploadPath = _jobSubmissionDirectoryProvider.GetJobSubmissionRemoteDirectory().TrimEnd('/') + @"/";
Log.Log(Level.Info, "DriverFolderPath: {0} DriverUploadPath: {1}", driverLocalFolderPath, driverUploadPath);
var archivePath = _resourceArchiveFileGenerator.CreateArchiveToUpload(driverLocalFolderPath);
var resourceDetailsOutputPath = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString("N"));
_javaLauncher.Launch(JavaClassNameForResourceUploader,
archivePath,
driverUploadPath,
resourceDetailsOutputPath);
return(ParseGeneratedOutputFile(resourceDetailsOutputPath));
}
public FileSystemJobResourceUploader GetJobResourceUploader()
{
var injector = TangFactory.GetTang().NewInjector();
JobSubmissionDirectoryProvider.GetJobSubmissionRemoteDirectory().Returns(AnyDriverResourceUploadPath);
FileSystem.GetFileStatus(new Uri(AnyUploadedResourceAbsoluteUri))
.Returns(new FileStatus(Epoch + TimeSpan.FromSeconds(AnyModificationTime), AnyResourceSize));
ResourceArchiveFileGenerator.CreateArchiveToUpload(AnyDriverLocalFolderPath)
.Returns(AnyLocalArchivePath);
FileSystem.CreateUriForPath(AnyDriverResourceUploadPath)
.Returns(new Uri(AnyScheme + AnyHost + AnyDriverResourceUploadPath));
FileSystem.CreateUriForPath(AnyUploadedResourcePath)
.Returns(new Uri(AnyUploadedResourceAbsoluteUri));
injector.BindVolatileInstance(GenericType <IJobSubmissionDirectoryProvider> .Class, JobSubmissionDirectoryProvider);
injector.BindVolatileInstance(GenericType <IResourceArchiveFileGenerator> .Class, ResourceArchiveFileGenerator);
injector.BindVolatileInstance(GenericType <IFileSystem> .Class, FileSystem);
return(injector.GetInstance <FileSystemJobResourceUploader>());
}
public FileSystemJobResourceUploader GetJobResourceUploader()
{
var injector = TangFactory.GetTang().NewInjector();
FileSystem.GetFileStatus(new Uri(AnyUploadedResourceAbsoluteUri))
.Returns(new FileStatus(Epoch + TimeSpan.FromMilliseconds(AnyModificationTime), AnyResourceSize));
FileSystem.GetFileStatus(new Uri(AnyJobFileResourceAbsoluteUri))
.Returns(new FileStatus(Epoch + TimeSpan.FromMilliseconds(AnyModificationTime), AnyResourceSize));
ResourceArchiveFileGenerator.CreateArchiveToUpload(AnyDriverLocalFolderPath)
.Returns(AnyLocalArchivePath);
FileSystem.CreateUriForPath(AnyDriverResourceUploadPath)
.Returns(new Uri(AnyScheme + AnyHost + AnyDriverResourceUploadPath));
FileSystem.CreateUriForPath(AnyUploadedResourcePath)
.Returns(new Uri(AnyUploadedResourceAbsoluteUri));
FileSystem.CreateUriForPath(AnyJobFileResourcePath)
.Returns(new Uri(AnyJobFileResourceAbsoluteUri));
IFile file = Substitute.For <IFile>();
file.Exists(Arg.Any <string>()).Returns(true);
injector.BindVolatileInstance(GenericType <IFile> .Class, file);
injector.BindVolatileInstance(GenericType <IResourceArchiveFileGenerator> .Class, ResourceArchiveFileGenerator);
injector.BindVolatileInstance(GenericType <IFileSystem> .Class, FileSystem);
return(injector.GetInstance <FileSystemJobResourceUploader>());
}