public async Task TestSparkBatchJob() { SparkBatchClient client = CreateClient(); // Submit the Spark job SparkBatchJobOptions createParams = SparkTestUtilities.CreateSparkJobRequestParameters(Recording, TestEnvironment); SparkBatchOperation createOperation = await client.StartCreateSparkBatchJobAsync(createParams); SparkBatchJob jobCreateResponse = await createOperation.WaitForCompletionAsync(); // Verify the Spark batch job completes successfully Assert.True("success".Equals(jobCreateResponse.State, StringComparison.OrdinalIgnoreCase) && jobCreateResponse.Result == SparkBatchJobResultType.Succeeded, string.Format( "Job: {0} did not return success. Current job state: {1}. Actual result: {2}. Error (if any): {3}", jobCreateResponse.Id, jobCreateResponse.State, jobCreateResponse.Result, string.Join(", ", jobCreateResponse.Errors ?? new List <SparkServiceError>()) ) ); // Get the list of Spark batch jobs and check that the submitted job exists List <SparkBatchJob> listJobResponse = await SparkTestUtilities.ListSparkBatchJobsAsync(client); Assert.NotNull(listJobResponse); Assert.IsTrue(listJobResponse.Any(job => job.Id == jobCreateResponse.Id)); }
public async Task TestSparkBatchJobCompletesWhenJobStarts() { SparkBatchClient client = CreateClient(); // Submit the Spark job SparkBatchJobOptions createParams = SparkTestUtilities.CreateSparkJobRequestParameters(Recording, TestEnvironment); SparkBatchOperation createOperation = await client.StartCreateSparkBatchJobAsync(createParams); SparkBatchJob jobCreateResponse = await createOperation.WaitForCompletionAsync(); // Verify the Spark batch job submission starts successfully Assert.True(LivyStates.Starting == jobCreateResponse.State || LivyStates.Running == jobCreateResponse.State || LivyStates.Success == jobCreateResponse.State, string.Format( "Job: {0} did not return success. Current job state: {1}. Error (if any): {2}", jobCreateResponse.Id, jobCreateResponse.State, string.Join(", ", jobCreateResponse.Errors ?? new List <SparkServiceError>()) ) ); // Get the list of Spark batch jobs and check that the submitted job exists List <SparkBatchJob> listJobResponse = await SparkTestUtilities.ListSparkBatchJobsAsync(client); Assert.NotNull(listJobResponse); Assert.IsTrue(listJobResponse.Any(job => job.Id == jobCreateResponse.Id)); }
public async Task SubmitSparkJobAsync() { #region Snippet:CreateSparkBatchClientAsync // Replace the strings below with the spark, endpoint, and file system information string sparkPoolName = "<my-spark-pool-name>"; /*@@*/ sparkPoolName = TestEnvironment.SparkPoolName; string endpoint = "<my-endpoint-url>"; /*@@*/ endpoint = TestEnvironment.EndpointUrl; string storageAccount = "<my-storage-account-name>"; /*@@*/ storageAccount = TestEnvironment.StorageAccountName; string fileSystem = "<my-storage-filesystem-name>"; /*@@*/ fileSystem = TestEnvironment.StorageFileSystemName; SparkBatchClient client = new SparkBatchClient(endpoint, sparkPoolName, new DefaultAzureCredential()); #endregion #region Snippet:SubmitSparkBatchJobAsync string name = $"batch-{Guid.NewGuid()}"; string file = string.Format("abfss://{0}@{1}.dfs.core.windows.net/samples/net/wordcount/wordcount.zip", fileSystem, storageAccount); SparkBatchJobOptions request = new SparkBatchJobOptions(name, file) { ClassName = "WordCount", Arguments = { string.Format("abfss://{0}@{1}.dfs.core.windows.net/samples/net/wordcount/shakespeare.txt", fileSystem, storageAccount), string.Format("abfss://{0}@{1}.dfs.core.windows.net/samples/net/wordcount/result/", fileSystem, storageAccount), }, DriverMemory = "28g", DriverCores = 4, ExecutorMemory = "28g", ExecutorCores = 4, ExecutorCount = 2 }; SparkBatchOperation createOperation = await client.StartCreateSparkBatchJobAsync(request); SparkBatchJob jobCreated = await createOperation.WaitForCompletionAsync(); #endregion #region Snippet:ListSparkBatchJobsAsync Response <SparkBatchJobCollection> jobs = client.GetSparkBatchJobs(); foreach (SparkBatchJob job in jobs.Value.Sessions) { Console.WriteLine(job.Name); } #endregion #region Snippet:GetSparkBatchJobAsync SparkBatchJob retrievedJob = await client.GetSparkBatchJobAsync(jobCreated.Id); Debug.WriteLine($"Job is returned with name {retrievedJob.Name} and state {retrievedJob.State}"); #endregion #region Snippet:CancelSparkBatchJobAsync Response operation = client.CancelSparkBatchJob(jobCreated.Id); #endregion }