/// <summary>
/// Gets the default cluster information.
/// </summary>
/// <param name="workers">The workers.</param>
/// <param name="nodeType">Type of the node.</param>
/// <returns>ClusterInfo.</returns>
private static ClusterInfo GetDefaultClusterInfo(
int workersMin,
int workersMax,
string nodeType)
{
var newCluster = ClusterInfo.GetNewClusterConfiguration()
.WithAutoScale(minWorkers: workersMin, maxWorkers: workersMax)
.WithPython3(true)
.WithNodeType(nodeType)
.WithRuntimeVersion(RuntimeVersions.Runtime_5_5);
return(newCluster);
}
private static async Task JobsApi(DatabricksClient client)
{
Console.WriteLine("Creating new job");
var newCluster = ClusterInfo.GetNewClusterConfiguration()
.WithNumberOfWorkers(3)
.WithNodeType(NodeTypes.Standard_D3_v2)
.WithRuntimeVersion(RuntimeVersions.Runtime_6_4_ESR);
Console.WriteLine($"Creating workspace {SampleWorkspacePath}");
await client.Workspace.Mkdirs(SampleWorkspacePath);
Console.WriteLine("Downloading sample notebook");
var content = await DownloadSampleNotebook();
Console.WriteLine($"Importing sample HTML notebook to {SampleNotebookPath}");
await client.Workspace.Import(SampleNotebookPath, ExportFormat.HTML, null,
content, true);
var schedule = new CronSchedule
{
QuartzCronExpression = "0 0 9 ? * MON-FRI",
TimezoneId = "Europe/London",
PauseStatus = PauseStatus.UNPAUSED
};
var jobSettings = JobSettings.GetNewNotebookJobSettings(
"Sample Job",
SampleNotebookPath,
null)
.WithNewCluster(newCluster)
.WithSchedule(schedule);
var jobId = await client.Jobs.Create(jobSettings);
Console.WriteLine("Job created: {0}", jobId);
// Adding email notifications and libraries.
await client.Jobs.Update(jobId, new JobSettings
{
EmailNotifications = new JobEmailNotifications
{
OnSuccess = new[] { "*****@*****.**" }
},
Libraries = new List <Library>
{
new MavenLibrary
{
MavenLibrarySpec = new MavenLibrarySpec
{
Coordinates = "com.microsoft.azure:synapseml_2.12:0.9.5"
}
}
}
});
// Removing email notifications and libraries.
await client.Jobs.Update(jobId, new JobSettings(), new[] { "email_notifications", "libraries" });
var jobWithClusterInfo = await client.Jobs.Get(jobId);
var existingSettings = jobWithClusterInfo.Settings;
var existingSchedule = existingSettings.Schedule;
existingSchedule.PauseStatus = PauseStatus.PAUSED;
existingSettings.Schedule = existingSchedule;
Console.WriteLine("Pausing job schedule");
await client.Jobs.Reset(jobId, existingSettings);
Console.WriteLine("Run now: {0}", jobId);
var runId = (await client.Jobs.RunNow(jobId, null)).RunId;
Console.WriteLine("Run Id: {0}", runId);
while (true)
{
var run = await client.Jobs.RunsGet(runId);
Console.WriteLine("[{0:s}] Run Id: {1}\tLifeCycleState: {2}\tStateMessage: {3}", DateTime.UtcNow, runId,
run.State.LifeCycleState, run.State.StateMessage);
if (run.State.LifeCycleState == RunLifeCycleState.PENDING ||
run.State.LifeCycleState == RunLifeCycleState.RUNNING ||
run.State.LifeCycleState == RunLifeCycleState.TERMINATING)
{
await Task.Delay(TimeSpan.FromSeconds(15));
}
else
{
break;
}
}
var viewItems = await client.Jobs.RunsExport(runId);
foreach (var viewItem in viewItems)
{
Console.WriteLine("Exported view item from run: " + viewItem.Name);
Console.WriteLine("====================");
Console.WriteLine(viewItem.Content.Substring(0, 100) + "...");
Console.WriteLine("====================");
}
Console.WriteLine("Deleting sample workspace");
await client.Workspace.Delete(SampleWorkspacePath, true);
}
private static async Task ClustersApi(DatabricksClient client)
{
Console.WriteLine("Listing node types (take 10)");
var nodeTypes = await client.Clusters.ListNodeTypes();
foreach (var nodeType in nodeTypes.Take(10))
{
Console.WriteLine($"\t{nodeType.NodeTypeId}\tMemory: {nodeType.MemoryMb} MB\tCores: {nodeType.NumCores}\tAvailable Quota: {nodeType.ClusterCloudProviderNodeInfo.AvailableCoreQuota}");
}
Console.WriteLine("Listing Databricks runtime versions");
var sparkVersions = await client.Clusters.ListSparkVersions();
foreach (var(key, name) in sparkVersions)
{
Console.WriteLine($"\t{key}\t\t{name}");
}
Console.WriteLine("Creating standard cluster");
var clusterConfig = ClusterInfo.GetNewClusterConfiguration("Sample cluster")
.WithRuntimeVersion(RuntimeVersions.Runtime_6_4_ESR)
.WithAutoTermination(30)
.WithClusterLogConf("dbfs:/logs/")
.WithNodeType(NodeTypes.Standard_D3_v2)
.WithClusterMode(ClusterMode.SingleNode);
clusterConfig.DockerImage = new DockerImage {
Url = "databricksruntime/standard:latest"
};
var clusterId = await client.Clusters.Create(clusterConfig);
var createdCluster = await client.Clusters.Get(clusterId);
var createdClusterConfig = JsonConvert.SerializeObject(createdCluster, Formatting.Indented);
Console.WriteLine("Created cluster config: ");
Console.WriteLine(createdClusterConfig);
while (true)
{
var state = await client.Clusters.Get(clusterId);
Console.WriteLine("[{0:s}] Cluster {1}\tState {2}\tMessage {3}", DateTime.UtcNow, clusterId,
state.State, state.StateMessage);
if (state.State == ClusterState.RUNNING || state.State == ClusterState.ERROR || state.State == ClusterState.TERMINATED)
{
break;
}
await Task.Delay(TimeSpan.FromSeconds(15));
}
Console.WriteLine("Deleting cluster {0}", clusterId);
await client.Clusters.Delete(clusterId);
Console.WriteLine("Creating HighConcurrency cluster");
clusterConfig = ClusterInfo.GetNewClusterConfiguration("Sample cluster")
.WithRuntimeVersion(RuntimeVersions.Runtime_6_4_ESR)
.WithAutoScale(3, 7)
.WithAutoTermination(30)
.WithClusterLogConf("dbfs:/logs/")
.WithNodeType(NodeTypes.Standard_D3_v2)
.WithClusterMode(ClusterMode.HighConcurrency)
.WithTableAccessControl(true);
clusterId = await client.Clusters.Create(clusterConfig);
createdCluster = await client.Clusters.Get(clusterId);
createdClusterConfig = JsonConvert.SerializeObject(createdCluster, Formatting.Indented);
Console.WriteLine("Created cluster config: ");
Console.WriteLine(createdClusterConfig);
while (true)
{
var state = await client.Clusters.Get(clusterId);
Console.WriteLine("[{0:s}] Cluster {1}\tState {2}\tMessage {3}", DateTime.UtcNow, clusterId,
state.State, state.StateMessage);
if (state.State == ClusterState.RUNNING || state.State == ClusterState.ERROR || state.State == ClusterState.TERMINATED)
{
break;
}
await Task.Delay(TimeSpan.FromSeconds(15));
}
Console.WriteLine("Deleting cluster {0}", clusterId);
await client.Clusters.Delete(clusterId);
Console.WriteLine("Getting all events from a test cluster");
const string testClusterId = "0530-210517-viced348";
EventsResponse eventsResponse = null;
var events = new List <ClusterEvent>();
do
{
var nextPage = eventsResponse?.NextPage;
eventsResponse = await client.Clusters.Events(
testClusterId,
nextPage?.StartTime,
nextPage?.EndTime,
nextPage?.Order,
nextPage?.EventTypes,
nextPage?.Offset,
nextPage?.Limit
);
events.AddRange(eventsResponse.Events);
} while (eventsResponse.HasNextPage);
Console.WriteLine("{0} events retrieved from cluster {1}.", events.Count, testClusterId);
Console.WriteLine("Top 10 events: ");
foreach (var e in events.Take(10))
{
Console.WriteLine("\t[{0:s}] {1}\t{2}", e.Timestamp, e.Type, e.Details.User);
}
}
private static async Task InstancePoolApi(DatabricksClient client)
{
Console.WriteLine("Creating Testing Instance Pool");
var poolAttributes = new InstancePoolAttributes
{
PoolName = "TestInstancePool",
PreloadedSparkVersions = new[] { RuntimeVersions.Runtime_6_4_ESR },
MinIdleInstances = 2,
MaxCapacity = 100,
IdleInstanceAutoTerminationMinutes = 15,
NodeTypeId = NodeTypes.Standard_D3_v2,
EnableElasticDisk = true,
DiskSpec = new DiskSpec
{
DiskCount = 2, DiskSize = 64, DiskType = DiskType.FromAzureDisk(AzureDiskVolumeType.STANDARD_LRS)
},
PreloadedDockerImages = new[]
{
new DockerImage {
Url = "databricksruntime/standard:latest"
}
},
AzureAttributes = new InstancePoolAzureAttributes {
Availability = AzureAvailability.SPOT_AZURE, SpotBidMaxPrice = -1
}
};
var poolId = await client.InstancePool.Create(poolAttributes).ConfigureAwait(false);
Console.WriteLine("Listing pools");
var pools = await client.InstancePool.List().ConfigureAwait(false);
foreach (var pool in pools)
{
Console.WriteLine($"\t{pool.PoolId}\t{pool.PoolName}\t{pool.State}");
}
Console.WriteLine("Getting created pool by poolId");
var targetPoolInfo = await client.InstancePool.Get(poolId).ConfigureAwait(false);
Console.WriteLine("Editing pool");
targetPoolInfo.MinIdleInstances = 3;
await client.InstancePool.Edit(poolId, targetPoolInfo).ConfigureAwait(false);
Console.WriteLine("Getting edited pool by poolId");
targetPoolInfo = await client.InstancePool.Get(poolId).ConfigureAwait(false);
Console.WriteLine($"MinIdleInstances: {targetPoolInfo.MinIdleInstances}");
Console.WriteLine("Creating a sample cluster in the pool.");
var clusterConfig = ClusterInfo.GetNewClusterConfiguration("Sample cluster")
.WithRuntimeVersion(RuntimeVersions.Runtime_7_3)
.WithAutoScale(3, 7)
.WithAutoTermination(30)
.WithClusterLogConf("dbfs:/logs/");
clusterConfig.InstancePoolId = poolId;
var clusterId = await client.Clusters.Create(clusterConfig);
var createdCluster = await client.Clusters.Get(clusterId);
Console.WriteLine($"Created cluster pool Id: {createdCluster.InstancePoolId}");
Console.WriteLine("Deleting pool");
await client.InstancePool.Delete(poolId);
Console.WriteLine("Deleting cluster");
await client.Clusters.Delete(clusterId);
}
private static async Task JobsApi(DatabricksClient client)
{
Console.WriteLine("Creating new job");
var newCluster = ClusterInfo.GetNewClusterConfiguration()
.WithNumberOfWorkers(3)
.WithPython3(true)
.WithNodeType(NodeTypes.Standard_D3_v2)
.WithRuntimeVersion(RuntimeVersions.Runtime_4_2_Scala_2_11);
Console.WriteLine($"Creating workspace {SampleWorkspacePath}");
await client.Workspace.Mkdirs(SampleWorkspacePath);
Console.WriteLine("Dowloading sample notebook");
var content = await DownloadSampleNotebook();
Console.WriteLine($"Importing sample HTML notebook to {SampleNotebookPath}");
await client.Workspace.Import(SampleNotebookPath, ExportFormat.HTML, null,
content, true);
var jobSettings = JobSettings.GetNewNotebookJobSettings(
"Sample Job",
SampleNotebookPath,
null)
.WithNewCluster(newCluster);
var jobId = await client.Jobs.Create(jobSettings);
Console.WriteLine("Job created: {0}", jobId);
Console.WriteLine("Run now: {0}", jobId);
var runId = (await client.Jobs.RunNow(jobId, null)).RunId;
Console.WriteLine("Run Id: {0}", runId);
while (true)
{
var run = await client.Jobs.RunsGet(runId);
Console.WriteLine("[{0:s}] Run Id: {1}\tLifeCycleState: {2}\tStateMessage: {3}", DateTime.UtcNow, runId,
run.State.LifeCycleState, run.State.StateMessage);
if (run.State.LifeCycleState == RunLifeCycleState.PENDING ||
run.State.LifeCycleState == RunLifeCycleState.RUNNING ||
run.State.LifeCycleState == RunLifeCycleState.TERMINATING)
{
await Task.Delay(TimeSpan.FromSeconds(15));
}
else
{
break;
}
}
var viewItems = await client.Jobs.RunsExport(runId);
foreach (var viewItem in viewItems)
{
Console.WriteLine("Exported view item from run: " + viewItem.Name);
Console.WriteLine("====================");
Console.WriteLine(viewItem.Content.Substring(0, 100) + "...");
Console.WriteLine("====================");
}
Console.WriteLine("Deleting sample workspace");
await client.Workspace.Delete(SampleWorkspacePath, true);
}
