public void CreateSparkBatchJob()
{
// Environment variable with the storage account associated with the Synapse workspace endpoint.
string storageAccount = TestEnvironment.StorageAccountName;
// Environment variable with the file system of the storage account.
string fileSystem = TestEnvironment.StorageFileSystemName;
#region Snippet:CreateBatchJob
string name = $"batchSample";
string file = string.Format("abfss://{0}@{1}.dfs.core.windows.net/samples/java/wordcount/wordcount.jar", fileSystem, storageAccount);
SparkBatchJobOptions options = new SparkBatchJobOptions(name: name, file: file)
{
ClassName = "WordCount",
Arguments = new List <string>
{
string.Format("abfss://{0}@{1}.dfs.core.windows.net/samples/java/wordcount/shakespeare.txt", fileSystem, storageAccount),
string.Format("abfss://{0}@{1}.dfs.core.windows.net/samples/java/wordcount/result/", fileSystem, storageAccount),
},
DriverMemory = "28g",
DriverCores = 4,
ExecutorMemory = "28g",
ExecutorCores = 4,
ExecutorCount = 2
};
SparkBatchJob jobCreated = batchClient.CreateSparkBatchJob(options);
#endregion
}
public void SubmitSparkJobSync()
{
// Environment variable with the Synapse workspace endpoint.
string endpoint = TestEnvironment.EndpointUrl;
// Environment variable with the Synapse Spark pool name.
string sparkPoolName = TestEnvironment.SparkPoolName;
// Environment variable with the ADLS Gen2 storage account associated with the Synapse workspace.
string storageAccount = TestEnvironment.StorageAccountName;
// Environment variable with the file system of ADLS Gen2 storage account associated with the Synapse workspace.
string fileSystem = TestEnvironment.StorageFileSystemName;
#region Snippet:CreateSparkBatchClient
SparkBatchClient client = new SparkBatchClient(new Uri(endpoint), sparkPoolName, new DefaultAzureCredential());
#endregion
#region Snippet:SubmitSparkBatchJob
string name = $"batch-{Guid.NewGuid()}";
string file = string.Format("abfss://{0}@{1}.dfs.core.windows.net/samples/net/wordcount/wordcount.zip", fileSystem, storageAccount);
SparkBatchJobOptions request = new SparkBatchJobOptions(name, file)
{
ClassName = "WordCount",
Arguments =
{
string.Format("abfss://{0}@{1}.dfs.core.windows.net/samples/net/wordcount/shakespeare.txt", fileSystem, storageAccount),
string.Format("abfss://{0}@{1}.dfs.core.windows.net/samples/net/wordcount/result/", fileSystem, storageAccount),
},
DriverMemory = "28g",
DriverCores = 4,
ExecutorMemory = "28g",
ExecutorCores = 4,
ExecutorCount = 2
};
SparkBatchJob jobCreated = client.CreateSparkBatchJob(request);
#endregion
#region Snippet:ListSparkBatchJobs
Response <SparkBatchJobCollection> jobs = client.GetSparkBatchJobs();
foreach (SparkBatchJob job in jobs.Value.Sessions)
{
Console.WriteLine(job.Name);
}
#endregion
#region Snippet:GetSparkBatchJob
SparkBatchJob retrievedJob = client.GetSparkBatchJob(jobCreated.Id);
Debug.WriteLine($"Job is returned with name {retrievedJob.Name} and state {retrievedJob.State}");
#endregion
#region Snippet:DeleteSparkBatchJob
Response operation = client.CancelSparkBatchJob(jobCreated.Id);
#endregion
}
public SparkBatchJob SubmitSparkBatchJob(SparkBatchJobOptions sparkBatchJobOptions, bool waitForCompletion)
{
var batch = _sparkBatchClient.CreateSparkBatchJob(sparkBatchJobOptions, detailed: true);
if (!waitForCompletion)
{
return(batch);
}
return(PollSparkBatchJobSubmission(batch));
}
public void SparkSample()
{
#region Snippet:CreateBatchClient
// Replace the string below with your actual endpoint url.
string endpoint = "<my-endpoint-url>";
/*@@*/ endpoint = TestEnvironment.EndpointUrl;
string sparkPoolName = TestEnvironment.SparkPoolName;
SparkBatchClient client = new SparkBatchClient(endpoint: new Uri(endpoint), sparkPoolName: sparkPoolName, credential: new DefaultAzureCredential());
#endregion
// Environment variable with the storage account associated with the Synapse workspace endpoint.
string storageAccount = TestEnvironment.StorageAccountName;
// Environment variable with the file system of the storage account.
string fileSystem = TestEnvironment.StorageFileSystemName;
#region Snippet:CreateBatchJob
string name = $"batchSample";
string file = string.Format("abfss://{0}@{1}.dfs.core.windows.net/samples/java/wordcount/wordcount.jar", fileSystem, storageAccount);
SparkBatchJobOptions options = new SparkBatchJobOptions(name: name, file: file)
{
ClassName = "WordCount",
Arguments =
{
string.Format("abfss://{0}@{1}.dfs.core.windows.net/samples/java/wordcount/shakespeare.txt", fileSystem, storageAccount),
string.Format("abfss://{0}@{1}.dfs.core.windows.net/samples/java/wordcount/result/", fileSystem, storageAccount),
},
DriverMemory = "28g",
DriverCores = 4,
ExecutorMemory = "28g",
ExecutorCores = 4,
ExecutorCount = 2
};
SparkBatchJob jobCreated = client.CreateSparkBatchJob(options);
#endregion
#region Snippet:ListSparkBatchJobs
Response <SparkBatchJobCollection> jobs = client.GetSparkBatchJobs();
foreach (SparkBatchJob job in jobs.Value.Sessions)
{
Console.WriteLine(job.Name);
}
#endregion
#region Snippet:DeleteSparkBatchJob
/*@@*/ int jobId = jobs.Value.Sessions.First().Id;
// Replace the integer below with your actual job ID.
//@@ string jobId = 0;
Response operation = client.CancelSparkBatchJob(jobId);
#endregion
}
public void SubmitSparkJobSync()
{
// Environment variable with the Synapse workspace endpoint.
string workspaceUrl = TestEnvironment.WorkspaceUrl;
// Environment variable with the Synapse Spark pool name.
string sparkPoolName = TestEnvironment.SparkPoolName;
// Environment variable with the ADLS Gen2 storage account associated with the Synapse workspace.
string storageAccount = TestEnvironment.StorageAccountName;
// Environment variable with the file system of ADLS Gen2 storage account associated with the Synapse workspace.
string fileSystem = TestEnvironment.StorageFileSystemName;
#region Snippet:SparkBatchSample1SparkBatchClient
SparkBatchClient client = new SparkBatchClient(new Uri(workspaceUrl), sparkPoolName, new DefaultAzureCredential());
#endregion
#region Snippet:SparkBatchSample1SubmitSparkJob
string name = $"batch-{Guid.NewGuid()}";
string file = string.Format("abfss://{0}@{1}.dfs.core.windows.net/samples/java/wordcount/wordcount.jar", fileSystem, storageAccount);
SparkBatchJobOptions request = new SparkBatchJobOptions(name, file)
{
ClassName = "WordCount",
Arguments = new List <string>
{
string.Format("abfss://{0}@{1}.dfs.core.windows.net/samples/java/wordcount/shakespeare.txt", fileSystem, storageAccount),
string.Format("abfss://{0}@{1}.dfs.core.windows.net/samples/java/wordcount/result/", fileSystem, storageAccount),
},
DriverMemory = "28g",
DriverCores = 4,
ExecutorMemory = "28g",
ExecutorCores = 4,
ExecutorCount = 2
};
SparkBatchJob jobCreated = client.CreateSparkBatchJob(request);
#endregion
#region Snippet:SparkBatchSample1GetSparkJob
SparkBatchJob job = client.GetSparkBatchJob(jobCreated.Id);
Debug.WriteLine($"Job is returned with name {job.Name} and state {job.State}");
#endregion
#region Snippet:SparkBatchSample1CancelSparkJob
Response operation = client.CancelSparkBatchJob(jobCreated.Id);
#endregion
}