private static async Task JobsApi(DatabricksClient client)
{
Console.WriteLine("Creating new job");
var newCluster = ClusterInfo.GetNewClusterConfiguration()
.WithNumberOfWorkers(3)
.WithNodeType(NodeTypes.Standard_D3_v2)
.WithRuntimeVersion(RuntimeVersions.Runtime_6_4_ESR);
Console.WriteLine($"Creating workspace {SampleWorkspacePath}");
await client.Workspace.Mkdirs(SampleWorkspacePath);
Console.WriteLine("Downloading sample notebook");
var content = await DownloadSampleNotebook();
Console.WriteLine($"Importing sample HTML notebook to {SampleNotebookPath}");
await client.Workspace.Import(SampleNotebookPath, ExportFormat.HTML, null,
content, true);
var schedule = new CronSchedule
{
QuartzCronExpression = "0 0 9 ? * MON-FRI",
TimezoneId = "Europe/London",
PauseStatus = PauseStatus.UNPAUSED
};
var jobSettings = JobSettings.GetNewNotebookJobSettings(
"Sample Job",
SampleNotebookPath,
null)
.WithNewCluster(newCluster)
.WithSchedule(schedule);
var jobId = await client.Jobs.Create(jobSettings);
Console.WriteLine("Job created: {0}", jobId);
// Adding email notifications and libraries.
await client.Jobs.Update(jobId, new JobSettings
{
EmailNotifications = new JobEmailNotifications
{
OnSuccess = new[] { "*****@*****.**" }
},
Libraries = new List <Library>
{
new MavenLibrary
{
MavenLibrarySpec = new MavenLibrarySpec
{
Coordinates = "com.microsoft.azure:synapseml_2.12:0.9.5"
}
}
}
});
// Removing email notifications and libraries.
await client.Jobs.Update(jobId, new JobSettings(), new[] { "email_notifications", "libraries" });
var jobWithClusterInfo = await client.Jobs.Get(jobId);
var existingSettings = jobWithClusterInfo.Settings;
var existingSchedule = existingSettings.Schedule;
existingSchedule.PauseStatus = PauseStatus.PAUSED;
existingSettings.Schedule = existingSchedule;
Console.WriteLine("Pausing job schedule");
await client.Jobs.Reset(jobId, existingSettings);
Console.WriteLine("Run now: {0}", jobId);
var runId = (await client.Jobs.RunNow(jobId, null)).RunId;
Console.WriteLine("Run Id: {0}", runId);
while (true)
{
var run = await client.Jobs.RunsGet(runId);
Console.WriteLine("[{0:s}] Run Id: {1}\tLifeCycleState: {2}\tStateMessage: {3}", DateTime.UtcNow, runId,
run.State.LifeCycleState, run.State.StateMessage);
if (run.State.LifeCycleState == RunLifeCycleState.PENDING ||
run.State.LifeCycleState == RunLifeCycleState.RUNNING ||
run.State.LifeCycleState == RunLifeCycleState.TERMINATING)
{
await Task.Delay(TimeSpan.FromSeconds(15));
}
else
{
break;
}
}
var viewItems = await client.Jobs.RunsExport(runId);
foreach (var viewItem in viewItems)
{
Console.WriteLine("Exported view item from run: " + viewItem.Name);
Console.WriteLine("====================");
Console.WriteLine(viewItem.Content.Substring(0, 100) + "...");
Console.WriteLine("====================");
}
Console.WriteLine("Deleting sample workspace");
await client.Workspace.Delete(SampleWorkspacePath, true);
}
private static async Task JobsApi(DatabricksClient client)
{
Console.WriteLine("Creating new job");
var newCluster = ClusterInfo.GetNewClusterConfiguration()
.WithNumberOfWorkers(3)
.WithPython3(true)
.WithNodeType(NodeTypes.Standard_D3_v2)
.WithRuntimeVersion(RuntimeVersions.Runtime_4_2_Scala_2_11);
Console.WriteLine($"Creating workspace {SampleWorkspacePath}");
await client.Workspace.Mkdirs(SampleWorkspacePath);
Console.WriteLine("Dowloading sample notebook");
var content = await DownloadSampleNotebook();
Console.WriteLine($"Importing sample HTML notebook to {SampleNotebookPath}");
await client.Workspace.Import(SampleNotebookPath, ExportFormat.HTML, null,
content, true);
var jobSettings = JobSettings.GetNewNotebookJobSettings(
"Sample Job",
SampleNotebookPath,
null)
.WithNewCluster(newCluster);
var jobId = await client.Jobs.Create(jobSettings);
Console.WriteLine("Job created: {0}", jobId);
Console.WriteLine("Run now: {0}", jobId);
var runId = (await client.Jobs.RunNow(jobId, null)).RunId;
Console.WriteLine("Run Id: {0}", runId);
while (true)
{
var run = await client.Jobs.RunsGet(runId);
Console.WriteLine("[{0:s}] Run Id: {1}\tLifeCycleState: {2}\tStateMessage: {3}", DateTime.UtcNow, runId,
run.State.LifeCycleState, run.State.StateMessage);
if (run.State.LifeCycleState == RunLifeCycleState.PENDING ||
run.State.LifeCycleState == RunLifeCycleState.RUNNING ||
run.State.LifeCycleState == RunLifeCycleState.TERMINATING)
{
await Task.Delay(TimeSpan.FromSeconds(15));
}
else
{
break;
}
}
var viewItems = await client.Jobs.RunsExport(runId);
foreach (var viewItem in viewItems)
{
Console.WriteLine("Exported view item from run: " + viewItem.Name);
Console.WriteLine("====================");
Console.WriteLine(viewItem.Content.Substring(0, 100) + "...");
Console.WriteLine("====================");
}
Console.WriteLine("Deleting sample workspace");
await client.Workspace.Delete(SampleWorkspacePath, true);
}