public SparkBatchJob SubmitSparkBatchJob(SparkBatchJobOptions sparkBatchJobOptions, bool waitForCompletion) { var batch = _sparkBatchClient.StartCreateSparkBatchJob(sparkBatchJobOptions, detailed: true); if (!waitForCompletion) { return(GetSparkBatchJob(int.Parse(batch.Id))); } return(batch.Poll().Value); }
public void SubmitSparkJobSync() { #region Snippet:CreateSparkBatchClient #if SNIPPET // Replace the strings below with the spark, endpoint, and file system information string sparkPoolName = "<my-spark-pool-name>"; string endpoint = "<my-endpoint-url>"; string storageAccount = "<my-storage-account-name>"; string fileSystem = "<my-storage-filesystem-name>"; #else string sparkPoolName = TestEnvironment.SparkPoolName; string endpoint = TestEnvironment.EndpointUrl; string storageAccount = TestEnvironment.StorageAccountName; string fileSystem = TestEnvironment.StorageFileSystemName; #endif SparkBatchClient client = new SparkBatchClient(new Uri(endpoint), sparkPoolName, new DefaultAzureCredential()); #endregion #region Snippet:SubmitSparkBatchJob string name = $"batch-{Guid.NewGuid()}"; string file = string.Format("abfss://{0}@{1}.dfs.core.windows.net/samples/net/wordcount/wordcount.zip", fileSystem, storageAccount); SparkBatchJobOptions request = new SparkBatchJobOptions(name, file) { ClassName = "WordCount", Arguments = { string.Format("abfss://{0}@{1}.dfs.core.windows.net/samples/net/wordcount/shakespeare.txt", fileSystem, storageAccount), string.Format("abfss://{0}@{1}.dfs.core.windows.net/samples/net/wordcount/result/", fileSystem, storageAccount), }, DriverMemory = "28g", DriverCores = 4, ExecutorMemory = "28g", ExecutorCores = 4, ExecutorCount = 2 }; SparkBatchOperation createOperation = client.StartCreateSparkBatchJob(request); while (!createOperation.HasCompleted) { System.Threading.Thread.Sleep(2000); createOperation.UpdateStatus(); } SparkBatchJob jobCreated = createOperation.Value; #endregion #region Snippet:ListSparkBatchJobs Response <SparkBatchJobCollection> jobs = client.GetSparkBatchJobs(); foreach (SparkBatchJob job in jobs.Value.Sessions) { Console.WriteLine(job.Name); } #endregion #region Snippet:GetSparkBatchJob SparkBatchJob retrievedJob = client.GetSparkBatchJob(jobCreated.Id); Debug.WriteLine($"Job is returned with name {retrievedJob.Name} and state {retrievedJob.State}"); #endregion #region Snippet:CancelSparkBatchJob Response operation = client.CancelSparkBatchJob(jobCreated.Id); #endregion }