public async Task TestSparkBatchJob()
{
SparkBatchClient client = CreateClient();
// Submit the Spark job
SparkBatchJobOptions createParams = SparkTestUtilities.CreateSparkJobRequestParameters(Recording, TestEnvironment);
SparkBatchOperation createOperation = await client.StartCreateSparkBatchJobAsync(createParams);
SparkBatchJob jobCreateResponse = await createOperation.WaitForCompletionAsync();
// Verify the Spark batch job completes successfully
Assert.True("success".Equals(jobCreateResponse.State, StringComparison.OrdinalIgnoreCase) && jobCreateResponse.Result == SparkBatchJobResultType.Succeeded,
string.Format(
"Job: {0} did not return success. Current job state: {1}. Actual result: {2}. Error (if any): {3}",
jobCreateResponse.Id,
jobCreateResponse.State,
jobCreateResponse.Result,
string.Join(", ", jobCreateResponse.Errors ?? new List <SparkServiceError>())
)
);
// Get the list of Spark batch jobs and check that the submitted job exists
List <SparkBatchJob> listJobResponse = await SparkTestUtilities.ListSparkBatchJobsAsync(client);
Assert.NotNull(listJobResponse);
Assert.IsTrue(listJobResponse.Any(job => job.Id == jobCreateResponse.Id));
}
public async Task TestSparkBatchJobCompletesWhenJobStarts()
{
SparkBatchClient client = CreateClient();
// Submit the Spark job
SparkBatchJobOptions createParams = SparkTestUtilities.CreateSparkJobRequestParameters(Recording, TestEnvironment);
SparkBatchOperation createOperation = await client.StartCreateSparkBatchJobAsync(createParams);
SparkBatchJob jobCreateResponse = await createOperation.WaitForCompletionAsync();
// Verify the Spark batch job submission starts successfully
Assert.True(LivyStates.Starting == jobCreateResponse.State || LivyStates.Running == jobCreateResponse.State || LivyStates.Success == jobCreateResponse.State,
string.Format(
"Job: {0} did not return success. Current job state: {1}. Error (if any): {2}",
jobCreateResponse.Id,
jobCreateResponse.State,
string.Join(", ", jobCreateResponse.Errors ?? new List <SparkServiceError>())
)
);
// Get the list of Spark batch jobs and check that the submitted job exists
List <SparkBatchJob> listJobResponse = await SparkTestUtilities.ListSparkBatchJobsAsync(client);
Assert.NotNull(listJobResponse);
Assert.IsTrue(listJobResponse.Any(job => job.Id == jobCreateResponse.Id));
}
public void SubmitSparkJobSync()
{
#region Snippet:CreateSparkBatchClient
#if SNIPPET
// Replace the strings below with the spark, endpoint, and file system information
string sparkPoolName = "<my-spark-pool-name>";
string endpoint = "<my-endpoint-url>";
string storageAccount = "<my-storage-account-name>";
string fileSystem = "<my-storage-filesystem-name>";
#else
string sparkPoolName = TestEnvironment.SparkPoolName;
string endpoint = TestEnvironment.EndpointUrl;
string storageAccount = TestEnvironment.StorageAccountName;
string fileSystem = TestEnvironment.StorageFileSystemName;
#endif
SparkBatchClient client = new SparkBatchClient(new Uri(endpoint), sparkPoolName, new DefaultAzureCredential());
#endregion
#region Snippet:SubmitSparkBatchJob
string name = $"batch-{Guid.NewGuid()}";
string file = string.Format("abfss://{0}@{1}.dfs.core.windows.net/samples/net/wordcount/wordcount.zip", fileSystem, storageAccount);
SparkBatchJobOptions request = new SparkBatchJobOptions(name, file)
{
ClassName = "WordCount",
Arguments =
{
string.Format("abfss://{0}@{1}.dfs.core.windows.net/samples/net/wordcount/shakespeare.txt", fileSystem, storageAccount),
string.Format("abfss://{0}@{1}.dfs.core.windows.net/samples/net/wordcount/result/", fileSystem, storageAccount),
},
DriverMemory = "28g",
DriverCores = 4,
ExecutorMemory = "28g",
ExecutorCores = 4,
ExecutorCount = 2
};
SparkBatchOperation createOperation = client.StartCreateSparkBatchJob(request);
while (!createOperation.HasCompleted)
{
System.Threading.Thread.Sleep(2000);
createOperation.UpdateStatus();
}
SparkBatchJob jobCreated = createOperation.Value;
#endregion
#region Snippet:ListSparkBatchJobs
Response <SparkBatchJobCollection> jobs = client.GetSparkBatchJobs();
foreach (SparkBatchJob job in jobs.Value.Sessions)
{
Console.WriteLine(job.Name);
}
#endregion
#region Snippet:GetSparkBatchJob
SparkBatchJob retrievedJob = client.GetSparkBatchJob(jobCreated.Id);
Debug.WriteLine($"Job is returned with name {retrievedJob.Name} and state {retrievedJob.State}");
#endregion
#region Snippet:CancelSparkBatchJob
Response operation = client.CancelSparkBatchJob(jobCreated.Id);
#endregion
}
