public async Task SubmitJobAsync()
{
Mock <JobController.JobControllerClient> mockGrpcClient = new Mock <JobController.JobControllerClient>(MockBehavior.Strict);
SubmitJobRequest expectedRequest = new SubmitJobRequest
{
ProjectId = "projectId-1969970175",
Region = "region-934795532",
Job = new Job(),
};
Job expectedResponse = new Job
{
DriverOutputResourceUri = "driverOutputResourceUri-542229086",
DriverControlFilesUri = "driverControlFilesUri207057643",
JobUuid = "jobUuid-1615012099",
};
mockGrpcClient.Setup(x => x.SubmitJobAsync(expectedRequest, It.IsAny <CallOptions>()))
.Returns(new Grpc.Core.AsyncUnaryCall <Job>(Task.FromResult(expectedResponse), null, null, null, null));
JobControllerClient client = new JobControllerClientImpl(mockGrpcClient.Object, null);
string projectId = "projectId-1969970175";
string region = "region-934795532";
Job job = new Job();
Job response = await client.SubmitJobAsync(projectId, region, job);
Assert.Same(expectedResponse, response);
mockGrpcClient.VerifyAll();
}
private async Task <SubmitJobResponse> SubmitRenderingJob(AWSCredentials credentials, RemoteCredentials config, string jobName, string bakeJobId)
{
var batchClient = new AmazonBatchClient(credentials);
var request = new SubmitJobRequest
{
JobDefinition = config.RenderJobDefinitionId,
JobName = jobName,
JobQueue = config.RenderJobQueueId,
Parameters = new Dictionary <string, string>
{
{ "job", jobName }
},
DependsOn = new List <JobDependency>()
};
if (!string.IsNullOrEmpty(bakeJobId))
{
request.DependsOn.Add(new JobDependency {
JobId = bakeJobId,
Type = ArrayJobDependency.N_TO_N
});
}
var response = await batchClient.SubmitJobAsync(request);
return(response);
}
/// <summary>
/// Submits a job to a cluster.
/// Documentation https://developers.google.com/dataproc/v1beta1/reference/jobs/submit
/// Generation Note: This does not always build corectly. Google needs to standardise things I need to figuer out which ones are wrong.
/// </summary>
/// <param name="service">Authenticated dataproc service.</param>
/// <param name="projectId">[Required] The ID of the Google Cloud Platform project that the job belongs to.</param>
/// <param name="body">A valid dataproc v1beta1 body.</param>
/// <returns>JobResponse</returns>
public static Job Submit(dataprocService service, string projectId, SubmitJobRequest body)
{
try
{
// Initial validation.
if (service == null)
{
throw new ArgumentNullException("service");
}
if (body == null)
{
throw new ArgumentNullException("body");
}
if (projectId == null)
{
throw new ArgumentNullException(projectId);
}
// Make the request.
return(service.Jobs.Submit(body, projectId).Execute());
}
catch (Exception ex)
{
throw new Exception("Request Jobs.Submit failed.", ex);
}
}
/// <summary>Snippet for SubmitJobAsOperation</summary>
public void SubmitJobAsOperationRequestObject()
{
// Snippet: SubmitJobAsOperation(SubmitJobRequest, CallSettings)
// Create client
JobControllerClient jobControllerClient = JobControllerClient.Create();
// Initialize request argument(s)
SubmitJobRequest request = new SubmitJobRequest
{
ProjectId = "",
Job = new Job(),
Region = "",
RequestId = "",
};
// Make the request
Operation <Job, JobMetadata> response = jobControllerClient.SubmitJobAsOperation(request);
// Poll until the returned long-running operation is complete
Operation <Job, JobMetadata> completedResponse = response.PollUntilCompleted();
// Retrieve the operation result
Job result = completedResponse.Result;
// Or get the name of the operation
string operationName = response.Name;
// This name can be stored, then the long-running operation retrieved later by name
Operation <Job, JobMetadata> retrievedResponse = jobControllerClient.PollOnceSubmitJobAsOperation(operationName);
// Check if the retrieved long-running operation has completed
if (retrievedResponse.IsCompleted)
{
// If it has completed, then access the result
Job retrievedResult = retrievedResponse.Result;
}
// End snippet
}
internal virtual SubmitJobResponse SubmitJob(SubmitJobRequest request)
{
var marshaller = new SubmitJobRequestMarshaller();
var unmarshaller = SubmitJobResponseUnmarshaller.Instance;
return(Invoke <SubmitJobRequest, SubmitJobResponse>(request, marshaller, unmarshaller));
}
public void SubmitJob()
{
Mock <JobController.JobControllerClient> mockGrpcClient = new Mock <JobController.JobControllerClient>(MockBehavior.Strict);
SubmitJobRequest expectedRequest = new SubmitJobRequest
{
ProjectId = "projectId-1969970175",
Region = "region-934795532",
Job = new Job(),
};
Job expectedResponse = new Job
{
DriverOutputResourceUri = "driverOutputResourceUri-542229086",
DriverControlFilesUri = "driverControlFilesUri207057643",
};
mockGrpcClient.Setup(x => x.SubmitJob(expectedRequest, It.IsAny <CallOptions>()))
.Returns(expectedResponse);
JobControllerClient client = new JobControllerClientImpl(mockGrpcClient.Object, null);
string projectId = "projectId-1969970175";
string region = "region-934795532";
Job job = new Job();
Job response = client.SubmitJob(projectId, region, job);
Assert.Same(expectedResponse, response);
mockGrpcClient.VerifyAll();
}
/// <summary>
/// Initiates the asynchronous execution of the SubmitJob operation.
/// </summary>
///
/// <param name="request">Container for the necessary parameters to execute the SubmitJob operation.</param>
/// <param name="cancellationToken">
/// A cancellation token that can be used by other objects or threads to receive notice of cancellation.
/// </param>
/// <returns>The task object representing the asynchronous operation.</returns>
/// <seealso href="http://docs.aws.amazon.com/goto/WebAPI/batch-2016-08-10/SubmitJob">REST API Reference for SubmitJob Operation</seealso>
public virtual Task <SubmitJobResponse> SubmitJobAsync(SubmitJobRequest request, System.Threading.CancellationToken cancellationToken = default(CancellationToken))
{
var marshaller = new SubmitJobRequestMarshaller();
var unmarshaller = SubmitJobResponseUnmarshaller.Instance;
return(InvokeAsync <SubmitJobRequest, SubmitJobResponse>(request, marshaller,
unmarshaller, cancellationToken));
}
/// <summary>
/// Initiates the asynchronous execution of the SubmitJob operation.
/// </summary>
///
/// <param name="request">Container for the necessary parameters to execute the SubmitJob operation.</param>
/// <param name="cancellationToken">
/// A cancellation token that can be used by other objects or threads to receive notice of cancellation.
/// </param>
/// <returns>The task object representing the asynchronous operation.</returns>
/// <seealso href="http://docs.aws.amazon.com/goto/WebAPI/batch-2016-08-10/SubmitJob">REST API Reference for SubmitJob Operation</seealso>
public virtual Task <SubmitJobResponse> SubmitJobAsync(SubmitJobRequest request, System.Threading.CancellationToken cancellationToken = default(CancellationToken))
{
var options = new InvokeOptions();
options.RequestMarshaller = SubmitJobRequestMarshaller.Instance;
options.ResponseUnmarshaller = SubmitJobResponseUnmarshaller.Instance;
return(InvokeAsync <SubmitJobResponse>(request, options, cancellationToken));
}
internal virtual SubmitJobResponse SubmitJob(SubmitJobRequest request)
{
var options = new InvokeOptions();
options.RequestMarshaller = SubmitJobRequestMarshaller.Instance;
options.ResponseUnmarshaller = SubmitJobResponseUnmarshaller.Instance;
return(Invoke <SubmitJobResponse>(request, options));
}
public async stt::Task SubmitJobRequestObjectAsync()
{
moq::Mock <JobController.JobControllerClient> mockGrpcClient = new moq::Mock <JobController.JobControllerClient>(moq::MockBehavior.Strict);
mockGrpcClient.Setup(x => x.CreateOperationsClient()).Returns(new moq::Mock <lro::Operations.OperationsClient>().Object);
SubmitJobRequest request = new SubmitJobRequest
{
ProjectId = "project_id43ad98b0",
Job = new Job(),
Region = "regionedb20d96",
RequestId = "request_id362c8df6",
};
Job expectedResponse = new Job
{
Reference = new JobReference(),
Placement = new JobPlacement(),
HadoopJob = new HadoopJob(),
SparkJob = new SparkJob(),
PysparkJob = new PySparkJob(),
HiveJob = new HiveJob(),
PigJob = new PigJob(),
Status = new JobStatus(),
YarnApplications =
{
new YarnApplication(),
},
SparkSqlJob = new SparkSqlJob(),
StatusHistory = { new JobStatus(), },
DriverControlFilesUri = "driver_control_files_uribe41e88b",
DriverOutputResourceUri = "driver_output_resource_urie2b470d7",
Labels =
{
{
"key8a0b6e3c",
"value60c16320"
},
},
Scheduling = new JobScheduling(),
SparkRJob = new SparkRJob(),
JobUuid = "job_uuid7530ffdd",
PrestoJob = new PrestoJob(),
Done = true,
};
mockGrpcClient.Setup(x => x.SubmitJobAsync(request, moq::It.IsAny <grpccore::CallOptions>())).Returns(new grpccore::AsyncUnaryCall <Job>(stt::Task.FromResult(expectedResponse), null, null, null, null));
JobControllerClient client = new JobControllerClientImpl(mockGrpcClient.Object, null);
Job responseCallSettings = await client.SubmitJobAsync(request, gaxgrpc::CallSettings.FromCancellationToken(st::CancellationToken.None));
xunit::Assert.Same(expectedResponse, responseCallSettings);
Job responseCancellationToken = await client.SubmitJobAsync(request, st::CancellationToken.None);
xunit::Assert.Same(expectedResponse, responseCancellationToken);
mockGrpcClient.VerifyAll();
}
private static bool IsSubmitJobRequestValid(SubmitJobRequest req)
{
if (string.IsNullOrEmpty(req.TenantId) || req.JobCriteria == null || req.SparkSubmitTaskParams == null || req.SparkSubmitTaskParams.Count == 0)
{
return(false);
}
var jobCriteria = req.JobCriteria;
if (string.IsNullOrEmpty(jobCriteria.JobPriority) || string.IsNullOrEmpty(jobCriteria.JobType))
{
return(false);
}
return(true);
}
public void SubmitJobRequestObject()
{
moq::Mock <JobController.JobControllerClient> mockGrpcClient = new moq::Mock <JobController.JobControllerClient>(moq::MockBehavior.Strict);
SubmitJobRequest request = new SubmitJobRequest
{
ProjectId = "project_id43ad98b0",
Job = new Job(),
Region = "regionedb20d96",
RequestId = "request_id362c8df6",
};
Job expectedResponse = new Job
{
Reference = new JobReference(),
Placement = new JobPlacement(),
HadoopJob = new HadoopJob(),
SparkJob = new SparkJob(),
PysparkJob = new PySparkJob(),
HiveJob = new HiveJob(),
PigJob = new PigJob(),
Status = new JobStatus(),
YarnApplications =
{
new YarnApplication(),
},
SparkSqlJob = new SparkSqlJob(),
StatusHistory = { new JobStatus(), },
DriverControlFilesUri = "driver_control_files_uribe41e88b",
DriverOutputResourceUri = "driver_output_resource_urie2b470d7",
Labels =
{
{
"key8a0b6e3c",
"value60c16320"
},
},
Scheduling = new JobScheduling(),
SparkRJob = new SparkRJob(),
JobUuid = "job_uuid7530ffdd",
PrestoJob = new PrestoJob(),
};
mockGrpcClient.Setup(x => x.SubmitJob(request, moq::It.IsAny <grpccore::CallOptions>())).Returns(expectedResponse);
JobControllerClient client = new JobControllerClientImpl(mockGrpcClient.Object, null);
Job response = client.SubmitJob(request);
xunit::Assert.Same(expectedResponse, response);
mockGrpcClient.VerifyAll();
}
/// <summary>
/// 用于提交一个作业
/// </summary>
/// <param name="req">参考<see cref="SubmitJobRequest"/></param>
/// <returns>参考<see cref="SubmitJobResponse"/>实例</returns>
public async Task <SubmitJobResponse> SubmitJob(SubmitJobRequest req)
{
JsonResponseModel <SubmitJobResponse> rsp = null;
try
{
var strResp = await this.InternalRequest(req, "SubmitJob");
rsp = JsonConvert.DeserializeObject <JsonResponseModel <SubmitJobResponse> >(strResp);
}
catch (JsonSerializationException e)
{
throw new TencentCloudSDKException(e.Message);
}
return(rsp.Response);
}
/// <summary>Snippet for SubmitJob</summary>
public void SubmitJob_RequestObject()
{
// Snippet: SubmitJob(SubmitJobRequest,CallSettings)
// Create client
JobControllerClient jobControllerClient = JobControllerClient.Create();
// Initialize request argument(s)
SubmitJobRequest request = new SubmitJobRequest
{
ProjectId = "",
Region = "",
Job = new Job(),
};
// Make the request
Job response = jobControllerClient.SubmitJob(request);
// End snippet
}
/// <summary>Snippet for SubmitJobAsync</summary>
public async Task SubmitJobAsync_RequestObject()
{
// Snippet: SubmitJobAsync(SubmitJobRequest,CallSettings)
// Additional: SubmitJobAsync(SubmitJobRequest,CancellationToken)
// Create client
JobControllerClient jobControllerClient = await JobControllerClient.CreateAsync();
// Initialize request argument(s)
SubmitJobRequest request = new SubmitJobRequest
{
ProjectId = "",
Region = "",
Job = new Job(),
};
// Make the request
Job response = await jobControllerClient.SubmitJobAsync(request);
// End snippet
}
public async Task <SubmitJobResponse> SubmitJobAsync(SubmitJobRequest req)
{
if (!Enum.TryParse <JobType>(req.JobCriteria.JobType, out JobType jobType))
{
throw new SjaasServiceValidationException("invalid job type specified: " + req.JobCriteria.JobType);
}
var job = new DataModel.Job
{
RefId = Guid.NewGuid().ToString(),
JobPriority = req.JobCriteria.JobPriority,
JobType = jobType,
TenantId = req.TenantId,
SparkSubmitTaskParams = req.SparkSubmitTaskParams,
JobStatus = JobStatus.Pending,
LastProcessedTime = DateTime.Now
};
await datastore.AddJobAsync(job);
return(new SubmitJobResponse(job.RefId));
}
private async Task <bool> ScheduleBatch(Guid id, string riskIds, ILambdaLogger logger)
{
try
{
// todo add batch
SubmitJobRequest request = new SubmitJobRequest();
request.Parameters = new Dictionary <string, string>
{
{ "fileId", id.ToString() }
};
//BatchClient.SubmitJobAsync()
return(true);
}
catch (Exception ex)
{
logger.LogLine("error on starting batch");
logger.LogLine(ex.Message);
logger.LogLine(ex.StackTrace);
throw;
}
}
public ActionResult <string> Post([FromBody] SubmitJobRequest request)
{
string jobId = Dispatch(request.JobName, request.JobParameters);
return(Accepted(jobId));
}
public async Task StartGalleryImport(string userId, string galleryId, Stream stream)
{
var tempFile = Path.GetTempFileName();
try
{
using (var fileStream = File.OpenWrite(tempFile))
{
Utilities.CopyStream(stream, fileStream);
}
var transferRequest = new TransferUtilityUploadRequest
{
BucketName = this._appOptions.MosaicStorageBucket,
Key = S3KeyManager.DetermineS3Key(userId, galleryId, S3KeyManager.ImageType.TileGallerySource),
FilePath = tempFile
};
await this._s3TransferUtility.UploadAsync(transferRequest).ConfigureAwait(false);
// Default verb is GET, and 60min validity should be more than sufficient!
var importUrl = this._s3Client.GetPreSignedURL(new GetPreSignedUrlRequest
{
BucketName = transferRequest.BucketName,
Key = transferRequest.Key,
Expires = DateTime.Now.AddHours(1)
});
var submitRequest = new SubmitJobRequest
{
JobQueue = this._appOptions.JobQueueArn,
JobDefinition = this._appOptions.JobDefinitionArn,
JobName = $"{Utilities.MakeSafeName(galleryId, 128)}",
ContainerOverrides = new ContainerOverrides
{
Environment = new List <Amazon.Batch.Model.KeyValuePair>
{
new Amazon.Batch.Model.KeyValuePair {
Name = Constants.ZIP_EXPANDER_BUCKET, Value = this._appOptions.MosaicStorageBucket
},
new Amazon.Batch.Model.KeyValuePair {
Name = Constants.ZIP_EXPANDER_DDB_TABLE, Value = this._appOptions.TableGallery
},
new Amazon.Batch.Model.KeyValuePair {
Name = Constants.ZIP_EXPANDER_USER_ID, Value = userId
},
new Amazon.Batch.Model.KeyValuePair {
Name = Constants.ZIP_EXPANDER_GALLERY_ID, Value = galleryId
},
new Amazon.Batch.Model.KeyValuePair {
Name = Constants.ZIP_EXPANDER_IMPORT_URL, Value = importUrl
}
}
}
};
await this._batchClient.SubmitJobAsync(submitRequest).ConfigureAwait(false);
}
finally
{
if (File.Exists(tempFile))
{
File.Delete(tempFile);
}
}
}
