/// <summary> /// Gets the default cluster information. /// </summary> /// <param name="workers">The workers.</param> /// <param name="nodeType">Type of the node.</param> /// <returns>ClusterInfo.</returns> private static ClusterInfo GetDefaultClusterInfo( int workersMin, int workersMax, string nodeType) { var newCluster = ClusterInfo.GetNewClusterConfiguration() .WithAutoScale(minWorkers: workersMin, maxWorkers: workersMax) .WithPython3(true) .WithNodeType(nodeType) .WithRuntimeVersion(RuntimeVersions.Runtime_5_5); return(newCluster); }
private static async Task JobsApi(DatabricksClient client) { Console.WriteLine("Creating new job"); var newCluster = ClusterInfo.GetNewClusterConfiguration() .WithNumberOfWorkers(3) .WithNodeType(NodeTypes.Standard_D3_v2) .WithRuntimeVersion(RuntimeVersions.Runtime_6_4_ESR); Console.WriteLine($"Creating workspace {SampleWorkspacePath}"); await client.Workspace.Mkdirs(SampleWorkspacePath); Console.WriteLine("Downloading sample notebook"); var content = await DownloadSampleNotebook(); Console.WriteLine($"Importing sample HTML notebook to {SampleNotebookPath}"); await client.Workspace.Import(SampleNotebookPath, ExportFormat.HTML, null, content, true); var schedule = new CronSchedule { QuartzCronExpression = "0 0 9 ? * MON-FRI", TimezoneId = "Europe/London", PauseStatus = PauseStatus.UNPAUSED }; var jobSettings = JobSettings.GetNewNotebookJobSettings( "Sample Job", SampleNotebookPath, null) .WithNewCluster(newCluster) .WithSchedule(schedule); var jobId = await client.Jobs.Create(jobSettings); Console.WriteLine("Job created: {0}", jobId); // Adding email notifications and libraries. await client.Jobs.Update(jobId, new JobSettings { EmailNotifications = new JobEmailNotifications { OnSuccess = new[] { "*****@*****.**" } }, Libraries = new List <Library> { new MavenLibrary { MavenLibrarySpec = new MavenLibrarySpec { Coordinates = "com.microsoft.azure:synapseml_2.12:0.9.5" } } } }); // Removing email notifications and libraries. await client.Jobs.Update(jobId, new JobSettings(), new[] { "email_notifications", "libraries" }); var jobWithClusterInfo = await client.Jobs.Get(jobId); var existingSettings = jobWithClusterInfo.Settings; var existingSchedule = existingSettings.Schedule; existingSchedule.PauseStatus = PauseStatus.PAUSED; existingSettings.Schedule = existingSchedule; Console.WriteLine("Pausing job schedule"); await client.Jobs.Reset(jobId, existingSettings); Console.WriteLine("Run now: {0}", jobId); var runId = (await client.Jobs.RunNow(jobId, null)).RunId; Console.WriteLine("Run Id: {0}", runId); while (true) { var run = await client.Jobs.RunsGet(runId); Console.WriteLine("[{0:s}] Run Id: {1}\tLifeCycleState: {2}\tStateMessage: {3}", DateTime.UtcNow, runId, run.State.LifeCycleState, run.State.StateMessage); if (run.State.LifeCycleState == RunLifeCycleState.PENDING || run.State.LifeCycleState == RunLifeCycleState.RUNNING || run.State.LifeCycleState == RunLifeCycleState.TERMINATING) { await Task.Delay(TimeSpan.FromSeconds(15)); } else { break; } } var viewItems = await client.Jobs.RunsExport(runId); foreach (var viewItem in viewItems) { Console.WriteLine("Exported view item from run: " + viewItem.Name); Console.WriteLine("===================="); Console.WriteLine(viewItem.Content.Substring(0, 100) + "..."); Console.WriteLine("===================="); } Console.WriteLine("Deleting sample workspace"); await client.Workspace.Delete(SampleWorkspacePath, true); }
private static async Task ClustersApi(DatabricksClient client) { Console.WriteLine("Listing node types (take 10)"); var nodeTypes = await client.Clusters.ListNodeTypes(); foreach (var nodeType in nodeTypes.Take(10)) { Console.WriteLine($"\t{nodeType.NodeTypeId}\tMemory: {nodeType.MemoryMb} MB\tCores: {nodeType.NumCores}\tAvailable Quota: {nodeType.ClusterCloudProviderNodeInfo.AvailableCoreQuota}"); } Console.WriteLine("Listing Databricks runtime versions"); var sparkVersions = await client.Clusters.ListSparkVersions(); foreach (var(key, name) in sparkVersions) { Console.WriteLine($"\t{key}\t\t{name}"); } Console.WriteLine("Creating standard cluster"); var clusterConfig = ClusterInfo.GetNewClusterConfiguration("Sample cluster") .WithRuntimeVersion(RuntimeVersions.Runtime_6_4_ESR) .WithAutoTermination(30) .WithClusterLogConf("dbfs:/logs/") .WithNodeType(NodeTypes.Standard_D3_v2) .WithClusterMode(ClusterMode.SingleNode); clusterConfig.DockerImage = new DockerImage { Url = "databricksruntime/standard:latest" }; var clusterId = await client.Clusters.Create(clusterConfig); var createdCluster = await client.Clusters.Get(clusterId); var createdClusterConfig = JsonConvert.SerializeObject(createdCluster, Formatting.Indented); Console.WriteLine("Created cluster config: "); Console.WriteLine(createdClusterConfig); while (true) { var state = await client.Clusters.Get(clusterId); Console.WriteLine("[{0:s}] Cluster {1}\tState {2}\tMessage {3}", DateTime.UtcNow, clusterId, state.State, state.StateMessage); if (state.State == ClusterState.RUNNING || state.State == ClusterState.ERROR || state.State == ClusterState.TERMINATED) { break; } await Task.Delay(TimeSpan.FromSeconds(15)); } Console.WriteLine("Deleting cluster {0}", clusterId); await client.Clusters.Delete(clusterId); Console.WriteLine("Creating HighConcurrency cluster"); clusterConfig = ClusterInfo.GetNewClusterConfiguration("Sample cluster") .WithRuntimeVersion(RuntimeVersions.Runtime_6_4_ESR) .WithAutoScale(3, 7) .WithAutoTermination(30) .WithClusterLogConf("dbfs:/logs/") .WithNodeType(NodeTypes.Standard_D3_v2) .WithClusterMode(ClusterMode.HighConcurrency) .WithTableAccessControl(true); clusterId = await client.Clusters.Create(clusterConfig); createdCluster = await client.Clusters.Get(clusterId); createdClusterConfig = JsonConvert.SerializeObject(createdCluster, Formatting.Indented); Console.WriteLine("Created cluster config: "); Console.WriteLine(createdClusterConfig); while (true) { var state = await client.Clusters.Get(clusterId); Console.WriteLine("[{0:s}] Cluster {1}\tState {2}\tMessage {3}", DateTime.UtcNow, clusterId, state.State, state.StateMessage); if (state.State == ClusterState.RUNNING || state.State == ClusterState.ERROR || state.State == ClusterState.TERMINATED) { break; } await Task.Delay(TimeSpan.FromSeconds(15)); } Console.WriteLine("Deleting cluster {0}", clusterId); await client.Clusters.Delete(clusterId); Console.WriteLine("Getting all events from a test cluster"); const string testClusterId = "0530-210517-viced348"; EventsResponse eventsResponse = null; var events = new List <ClusterEvent>(); do { var nextPage = eventsResponse?.NextPage; eventsResponse = await client.Clusters.Events( testClusterId, nextPage?.StartTime, nextPage?.EndTime, nextPage?.Order, nextPage?.EventTypes, nextPage?.Offset, nextPage?.Limit ); events.AddRange(eventsResponse.Events); } while (eventsResponse.HasNextPage); Console.WriteLine("{0} events retrieved from cluster {1}.", events.Count, testClusterId); Console.WriteLine("Top 10 events: "); foreach (var e in events.Take(10)) { Console.WriteLine("\t[{0:s}] {1}\t{2}", e.Timestamp, e.Type, e.Details.User); } }
private static async Task InstancePoolApi(DatabricksClient client) { Console.WriteLine("Creating Testing Instance Pool"); var poolAttributes = new InstancePoolAttributes { PoolName = "TestInstancePool", PreloadedSparkVersions = new[] { RuntimeVersions.Runtime_6_4_ESR }, MinIdleInstances = 2, MaxCapacity = 100, IdleInstanceAutoTerminationMinutes = 15, NodeTypeId = NodeTypes.Standard_D3_v2, EnableElasticDisk = true, DiskSpec = new DiskSpec { DiskCount = 2, DiskSize = 64, DiskType = DiskType.FromAzureDisk(AzureDiskVolumeType.STANDARD_LRS) }, PreloadedDockerImages = new[] { new DockerImage { Url = "databricksruntime/standard:latest" } }, AzureAttributes = new InstancePoolAzureAttributes { Availability = AzureAvailability.SPOT_AZURE, SpotBidMaxPrice = -1 } }; var poolId = await client.InstancePool.Create(poolAttributes).ConfigureAwait(false); Console.WriteLine("Listing pools"); var pools = await client.InstancePool.List().ConfigureAwait(false); foreach (var pool in pools) { Console.WriteLine($"\t{pool.PoolId}\t{pool.PoolName}\t{pool.State}"); } Console.WriteLine("Getting created pool by poolId"); var targetPoolInfo = await client.InstancePool.Get(poolId).ConfigureAwait(false); Console.WriteLine("Editing pool"); targetPoolInfo.MinIdleInstances = 3; await client.InstancePool.Edit(poolId, targetPoolInfo).ConfigureAwait(false); Console.WriteLine("Getting edited pool by poolId"); targetPoolInfo = await client.InstancePool.Get(poolId).ConfigureAwait(false); Console.WriteLine($"MinIdleInstances: {targetPoolInfo.MinIdleInstances}"); Console.WriteLine("Creating a sample cluster in the pool."); var clusterConfig = ClusterInfo.GetNewClusterConfiguration("Sample cluster") .WithRuntimeVersion(RuntimeVersions.Runtime_7_3) .WithAutoScale(3, 7) .WithAutoTermination(30) .WithClusterLogConf("dbfs:/logs/"); clusterConfig.InstancePoolId = poolId; var clusterId = await client.Clusters.Create(clusterConfig); var createdCluster = await client.Clusters.Get(clusterId); Console.WriteLine($"Created cluster pool Id: {createdCluster.InstancePoolId}"); Console.WriteLine("Deleting pool"); await client.InstancePool.Delete(poolId); Console.WriteLine("Deleting cluster"); await client.Clusters.Delete(clusterId); }
private static async Task JobsApi(DatabricksClient client) { Console.WriteLine("Creating new job"); var newCluster = ClusterInfo.GetNewClusterConfiguration() .WithNumberOfWorkers(3) .WithPython3(true) .WithNodeType(NodeTypes.Standard_D3_v2) .WithRuntimeVersion(RuntimeVersions.Runtime_4_2_Scala_2_11); Console.WriteLine($"Creating workspace {SampleWorkspacePath}"); await client.Workspace.Mkdirs(SampleWorkspacePath); Console.WriteLine("Dowloading sample notebook"); var content = await DownloadSampleNotebook(); Console.WriteLine($"Importing sample HTML notebook to {SampleNotebookPath}"); await client.Workspace.Import(SampleNotebookPath, ExportFormat.HTML, null, content, true); var jobSettings = JobSettings.GetNewNotebookJobSettings( "Sample Job", SampleNotebookPath, null) .WithNewCluster(newCluster); var jobId = await client.Jobs.Create(jobSettings); Console.WriteLine("Job created: {0}", jobId); Console.WriteLine("Run now: {0}", jobId); var runId = (await client.Jobs.RunNow(jobId, null)).RunId; Console.WriteLine("Run Id: {0}", runId); while (true) { var run = await client.Jobs.RunsGet(runId); Console.WriteLine("[{0:s}] Run Id: {1}\tLifeCycleState: {2}\tStateMessage: {3}", DateTime.UtcNow, runId, run.State.LifeCycleState, run.State.StateMessage); if (run.State.LifeCycleState == RunLifeCycleState.PENDING || run.State.LifeCycleState == RunLifeCycleState.RUNNING || run.State.LifeCycleState == RunLifeCycleState.TERMINATING) { await Task.Delay(TimeSpan.FromSeconds(15)); } else { break; } } var viewItems = await client.Jobs.RunsExport(runId); foreach (var viewItem in viewItems) { Console.WriteLine("Exported view item from run: " + viewItem.Name); Console.WriteLine("===================="); Console.WriteLine(viewItem.Content.Substring(0, 100) + "..."); Console.WriteLine("===================="); } Console.WriteLine("Deleting sample workspace"); await client.Workspace.Delete(SampleWorkspacePath, true); }