public async Task TestSparkBatchJob()
{
SparkBatchClient client = CreateClient();
// Submit the Spark job
SparkBatchJobOptions createParams = SparkTestUtilities.CreateSparkJobRequestParameters(Recording, TestEnvironment);
SparkBatchOperation createOperation = await client.StartCreateSparkBatchJobAsync(createParams);
SparkBatchJob jobCreateResponse = await createOperation.WaitForCompletionAsync();
// Verify the Spark batch job completes successfully
Assert.True("success".Equals(jobCreateResponse.State, StringComparison.OrdinalIgnoreCase) && jobCreateResponse.Result == SparkBatchJobResultType.Succeeded,
string.Format(
"Job: {0} did not return success. Current job state: {1}. Actual result: {2}. Error (if any): {3}",
jobCreateResponse.Id,
jobCreateResponse.State,
jobCreateResponse.Result,
string.Join(", ", jobCreateResponse.Errors ?? new List <SparkServiceError>())
)
);
// Get the list of Spark batch jobs and check that the submitted job exists
List <SparkBatchJob> listJobResponse = await SparkTestUtilities.ListSparkBatchJobsAsync(client);
Assert.NotNull(listJobResponse);
Assert.IsTrue(listJobResponse.Any(job => job.Id == jobCreateResponse.Id));
}
public async Task TestSparkBatchJobCompletesWhenJobStarts()
{
SparkBatchClient client = CreateClient();
// Submit the Spark job
SparkBatchJobOptions createParams = SparkTestUtilities.CreateSparkJobRequestParameters(Recording, TestEnvironment);
SparkBatchOperation createOperation = await client.StartCreateSparkBatchJobAsync(createParams);
SparkBatchJob jobCreateResponse = await createOperation.WaitForCompletionAsync();
// Verify the Spark batch job submission starts successfully
Assert.True(LivyStates.Starting == jobCreateResponse.State || LivyStates.Running == jobCreateResponse.State || LivyStates.Success == jobCreateResponse.State,
string.Format(
"Job: {0} did not return success. Current job state: {1}. Error (if any): {2}",
jobCreateResponse.Id,
jobCreateResponse.State,
string.Join(", ", jobCreateResponse.Errors ?? new List <SparkServiceError>())
)
);
// Get the list of Spark batch jobs and check that the submitted job exists
List <SparkBatchJob> listJobResponse = await SparkTestUtilities.ListSparkBatchJobsAsync(client);
Assert.NotNull(listJobResponse);
Assert.IsTrue(listJobResponse.Any(job => job.Id == jobCreateResponse.Id));
}