//https://docs.azuredatabricks.net/api/latest/jobs.html#runs-submit
public async Task <DatabricksRunNowResponse> JobsRunsSubmitJarTaskAsync(RunsSubmitJarTaskRequest runsSubmitJarTaskRequest)
{
var request = new RestRequest("2.0/jobs/runs/submit", Method.POST);
request.AddHeader("Authorization", $"Basic {_authHeader}");
request.AddJsonBody(runsSubmitJarTaskRequest);
var response = await _client.ExecuteTaskAsync <DatabricksRunNowResponse>(request);
var dbResponse = JsonConvert.DeserializeObject <DatabricksRunNowResponse>(response.Content);
return(dbResponse);
}
private async Task <int?> SubmitJarJobAsync(FileInfo dbfsFile)
{
this.Status = $"Creating the Spark job using '2.0/jobs/runs/submit'";
// =====================================================================
// EXAMPLE REQUEST
// =====================================================================
//{
// "run_name": "my spark task",
// "new_cluster":
// {
// "spark_version": "3.4.x-scala2.11",
// "node_type_id": "Standard_D3_v2",
// "num_workers": 10
// },
// "libraries": [
// {
// "jar": "dbfs:/my-jar.jar"
// }
// ],
// "timeout_seconds": 3600,
// "spark_jar_task": {
// "main_class_name": "com.databricks.ComputeModels",
// "parameters" : ["10"]
// }
//}
var datePart = DateTime.Now.ToShortDateString().Replace("/", "");
var timePart = DateTime.Now.ToShortTimeString().Replace(":", "");
var request = new RunsSubmitJarTaskRequest()
{
run_name = $"JobsPickAndRunJarViewModel_{datePart}_{timePart}",
new_cluster = new NewCluster
{
// see api/2.0/clusters/spark-versions
spark_version = "4.0.x-scala2.11",
// see api/2.0/clusters/list-node-types
node_type_id = "Standard_F4s",
num_workers = 2
},
libraries = new List <Library>
{
new Library {
jar = $"dbfs:/{dbfsFile.Name}"
}
},
timeout_seconds = 3600,
spark_jar_task = new SparkJarTask
{
main_class_name = "SparkApp",
parameters = new List <string>()
{
"10"
}
}
};
var response = await _databricksWebApiClient.JobsRunsSubmitJarTaskAsync(request);
return(response.RunId);
}
