private static async Task DbfsApi(DatabricksClient client)
        {
            Console.WriteLine("Listing directories under dbfs:/");
            var result = await client.Dbfs.List("/");

            foreach (var fileInfo in result)
            {
                Console.WriteLine(fileInfo.IsDirectory ? "[{0}]\t{1}" : "{0}\t{1}", fileInfo.Path, fileInfo.FileSize);
            }

            Console.WriteLine("Uploading a file");
            var uploadPath = "/test/" + Guid.NewGuid() + ".txt";

            using (var ms = new MemoryStream())
            {
                using (var httpClient = new HttpClient())
                {
                    var response = await httpClient.GetAsync("https://norvig.com/big.txt",
                                                             HttpCompletionOption.ResponseHeadersRead);

                    await response.Content.CopyToAsync(ms);
                }

                await client.Dbfs.Upload(uploadPath, true, ms);
            }

            Console.WriteLine("Getting info of the uploaded file");
            var uploadedFile = await client.Dbfs.GetStatus(uploadPath);

            Console.WriteLine("Path: {0}\tSize: {1}", uploadedFile.Path, uploadedFile.FileSize);

            Console.WriteLine("Deleting uploaded file");
            await client.Dbfs.Delete(uploadPath, false);
        }
Ejemplo n.º 2
0
        /// <summary>
        /// Creates a <see cref="DatabricksClient"/> instance with a <see cref="IJobsApi"/> implementation that returns the given <paramref name="runs"/> and <paramref name="jobs"/>.
        /// </summary>
        /// <param name="runs">The stubbed runs the <see cref="IJobsApi.RunsList"/> should return.</param>
        /// <param name="jobs">The stubbed jobs that the <see cref="IJobsApi.List"/> should return.</param>
        /// <exception cref="System.ArgumentNullException">Thrown when <paramref name="runs"/> or <paramref name="jobs"/> is <c>null</c>.</exception>
        public static DatabricksClient Create(IEnumerable <Run> runs, IEnumerable <Job> jobs)
        {
            Guard.NotNull(runs, nameof(runs));
            Guard.NotNull(jobs, nameof(jobs));

            var jobsStub = new Mock <IJobsApi>();

            jobsStub.Setup(j => j.RunsList(null, It.IsAny <int>(), It.IsAny <int>(), It.IsAny <bool>(), It.IsAny <bool>(), It.IsAny <CancellationToken>()))
            .ReturnsAsync(new RunList {
                Runs = runs
            });

            jobsStub.Setup(j => j.List(It.IsAny <CancellationToken>()))
            .ReturnsAsync(jobs);

            var client = DatabricksClient.CreateClient(
                clusterApi: Mock.Of <IClustersApi>(),
                jobsApi: jobsStub.Object,
                dbfsApi: Mock.Of <IDbfsApi>(),
                secretsApi: Mock.Of <ISecretsApi>(),
                groupsApi: Mock.Of <IGroupsApi>(),
                librariesApi: Mock.Of <ILibrariesApi>(),
                tokenApi: Mock.Of <ITokenApi>(),
                workspaceApi: Mock.Of <IWorkspaceApi>(),
                instancePoolApi: Mock.Of <IInstancePoolApi>());

            return(client);
        }
Ejemplo n.º 3
0
        private static async Task TokenApi(DatabricksClient client)
        {
            Console.WriteLine("Creating token without expiry");
            var(tokenValue, tokenInfo) = await client.Token.Create(null, "Sample token");

            Console.WriteLine("Token value: {0}", tokenValue);
            Console.WriteLine("Token Id {0}", tokenInfo.TokenId);
            Console.WriteLine("Token comment {0}", tokenInfo.Comment);
            Console.WriteLine("Token creation time {0:s}", tokenInfo.CreationTime);
            Console.WriteLine("Token expiry time {0:s}", tokenInfo.ExpiryTime);
            Console.WriteLine("Deleting token");
            await client.Token.Revoke(tokenInfo.TokenId);

            Console.WriteLine("Creating token with expiry");
            (tokenValue, tokenInfo) = await client.Token.Create(3600, "Sample token");

            Console.WriteLine("Token value: {0}", tokenValue);
            Console.WriteLine("Token comment {0}", tokenInfo.Comment);
            Console.WriteLine("Token creation time {0:s}", tokenInfo.CreationTime);
            Console.WriteLine("Token expiry time {0:s}", tokenInfo.ExpiryTime);
            Console.WriteLine("Deleting token");
            await client.Token.Revoke(tokenInfo.TokenId);

            Console.WriteLine("Listing tokens");
            var tokens = await client.Token.List();

            foreach (var token in tokens)
            {
                Console.WriteLine("Token Id {0}\tComment {1}", token.TokenId, token.Comment);
            }
        }
Ejemplo n.º 4
0
        public void TestParseDatabricksJobState()
        {
            var expectedValue = JobState.Running;
            var actualValue   = DatabricksClient.ParseDatabricksJobState("RUNNING");

            Assert.AreEqual(expectedValue, actualValue, "State mismatch when job state is RUNNING");

            expectedValue = JobState.Starting;
            actualValue   = DatabricksClient.ParseDatabricksJobState("PENDING");
            Assert.AreEqual(expectedValue, actualValue, "State mismatch when job state is PENDING");

            expectedValue = JobState.Error;
            actualValue   = DatabricksClient.ParseDatabricksJobState("INTERNAL_ERROR");
            Assert.AreEqual(expectedValue, actualValue, "State mismatch when job state is INTERNAL_ERROR");

            expectedValue = JobState.Idle;
            actualValue   = DatabricksClient.ParseDatabricksJobState("SKIPPED");
            Assert.AreEqual(expectedValue, actualValue, "State mismatch when job state is SKIPPED");

            actualValue = DatabricksClient.ParseDatabricksJobState("TERMINATING");
            Assert.AreEqual(expectedValue, actualValue, "State mismatch when job state is TERMINATING");

            actualValue = DatabricksClient.ParseDatabricksJobState("TERMINATED");
            Assert.AreEqual(expectedValue, actualValue, "State mismatch when job state is TERMINATED");
        }
Ejemplo n.º 5
0
 /// <summary>
 /// Deletes jobs.
 /// </summary>
 /// <param name="jobId">The job identifier.</param>
 public void JobsDelete(long jobId)
 {
     using (var client = DatabricksClient.CreateClient(this.baseUrl, this.token))
     {
         client.Jobs.Delete(jobId).Wait();
     }
 }
Ejemplo n.º 6
0
 /// <summary>
 /// Deletes the DBFS path.
 /// </summary>
 /// <param name="path">The path.</param>
 /// <param name="recurse">if set to <c>true</c> [recurse].</param>
 public void DbfsDelete(string path, bool recurse)
 {
     using (var client = DatabricksClient.CreateClient(this.baseUrl, this.token))
     {
         client.Dbfs.Delete(path, recurse);
     }
 }
Ejemplo n.º 7
0
        public static async Task Main(string[] args)
        {
            if (args.Length < 2)
            {
                await Console.Error.WriteLineAsync("Usage: <Azure databricks base url> <access token>");

                return;
            }

            var baseUrl = args[0];
            var token   = args[1];

            Console.WriteLine("Creating client");
            using (var client = DatabricksClient.CreateClient(baseUrl, token))
            {
                await WorkspaceApi(client);
                await LibrariesApi(client);
                await SecretsApi(client);
                await TokenApi(client);
                await GroupsApi(client);
                await DbfsApi(client);
                await JobsApi(client);
                await ClustersApi(client);
                await InstancePoolApi(client);
            }

            Console.WriteLine("Press enter to exit");
            Console.ReadLine();
        }
Ejemplo n.º 8
0
        public async Task GetFinishedJobRuns_OutsideTimeWindow_ReturnsNoFinishedJobs()
        {
            // Arrange
            DateTimeOffset startWindow = BogusGenerator.Date.RecentOffset();
            DateTimeOffset endWindow   = BogusGenerator.Date.SoonOffset();

            IEnumerable <Run> includedRuns = CreateRandomRuns(startWindow, endWindow);
            IEnumerable <Run> tooLateRuns  = CreateRandomRuns(endWindow, BogusGenerator.Date.FutureOffset());
            IEnumerable <Run> allRuns      = includedRuns.Concat(tooLateRuns);
            IEnumerable <Job> jobs         = includedRuns.Select(r => new Job
            {
                JobId    = r.JobId,
                Settings = new JobSettings {
                    Name = Guid.NewGuid().ToString()
                }
            }).ToArray();

            DatabricksClient client = DatabricksClientFactory.Create(allRuns, jobs);
            var provider            = new DatabricksInfoProvider(client, NullLogger.Instance);

            // Act
            IEnumerable <JobRun> finishedJobs = await provider.GetFinishedJobRunsAsync(BogusGenerator.Date.PastOffset(), startWindow);

            // Assert
            Assert.Empty(finishedJobs);
        }
Ejemplo n.º 9
0
 /// <summary>
 /// Gets the output for the job.
 /// </summary>
 /// <param name="runId">The run identifier.</param>
 /// <returns>System.ValueTuple&lt;System.String, System.String, Run&gt;.</returns>
 public (string, string, Run) JobsRunsGetOutput(long runId)
 {
     using (var client = DatabricksClient.CreateClient(this.baseUrl, this.token))
     {
         return(client.Jobs.RunsGetOutput(runId).Result);
     }
 }
Ejemplo n.º 10
0
 public long JobsRunSubmit(RunOnceSettings settings)
 {
     using (var client = DatabricksClient.CreateClient(this.baseUrl, this.token))
     {
         return(client.Jobs.RunSubmit(settings).Result);
     }
 }
Ejemplo n.º 11
0
 public async Task RunDatabricks()
 {
     using (var client = DatabricksClient.CreateClient(Settings.Instance.Databricks.BaseUrl, Settings.Instance.Databricks.Token))
     {
         var jobId = await client.Jobs.RunNow(51, null);
     }
 }
Ejemplo n.º 12
0
        public async Task MinimumAzureFunctionsDatabricksProject_WithEmbeddedTimer_ReportsAsMetricPeriodically()
        {
            ApplicationInsightsConfig applicationInsightsConfig = _config.GetApplicationInsightsConfig();
            var parameters = RunParameters.CreateNotebookParams(Enumerable.Empty <KeyValuePair <string, string> >());

            using (var project = AzureFunctionsDatabricksProject.StartNew(_config, _outputWriter))
            {
                using (var client = DatabricksClient.CreateClient(project.DatabricksConfig.BaseUrl, project.DatabricksConfig.SecurityToken))
                {
                    // Act
                    await client.Jobs.RunNow(project.DatabricksConfig.JobId, parameters);
                    await WaitUntilDatabricksJobRunIsCompleted(client, project.DatabricksConfig.JobId);
                }
            }

            // Assert
            using (ApplicationInsightsDataClient client = CreateApplicationInsightsClient(applicationInsightsConfig.ApiKey))
            {
                await RetryAssertUntilTelemetryShouldBeAvailableAsync(async() =>
                {
                    const string past10MinFilter = "PT0.1H";
                    var bodySchema = new MetricsPostBodySchema(
                        id: Guid.NewGuid().ToString(),
                        parameters: new MetricsPostBodySchemaParameters($"customMetrics/{applicationInsightsConfig.MetricName}", timespan: past10MinFilter));

                    IList <MetricsResultsItem> results =
                        await client.Metrics.GetMultipleAsync(applicationInsightsConfig.ApplicationId, new List <MetricsPostBodySchema> {
                        bodySchema
                    });

                    Assert.NotEmpty(results);
                    Assert.All(results, result => Assert.NotNull(result.Body.Value));
                }, timeout : TimeSpan.FromMinutes(2));
            }
        }
Ejemplo n.º 13
0
 /// <summary>
 /// Runs the job now.
 /// </summary>
 /// <param name="jobId">The job identifier.</param>
 /// <param name="runParameters">The run parameters.</param>
 /// <returns>RunIdentifier.</returns>
 public RunIdentifier JobsRunNow(long jobId, RunParameters runParameters)
 {
     using (var client = DatabricksClient.CreateClient(this.baseUrl, this.token))
     {
         return(client.Jobs.RunNow(jobId, runParameters).Result);
     }
 }
Ejemplo n.º 14
0
 /// <summary>
 /// Imports the workspace.
 /// </summary>
 /// <param name="path">The path.</param>
 /// <param name="format">The format.</param>
 /// <param name="language">The language.</param>
 /// <param name="content">The content.</param>
 /// <param name="overwrite">if set to <c>true</c> [overwrite].</param>
 public void WorkspaceImport(string path, ExportFormat format, Language?language, byte[] content, bool overwrite)
 {
     using (var client = DatabricksClient.CreateClient(this.baseUrl, this.token))
     {
         client.Workspace.Import(path, format, language, content, overwrite).Wait();
     }
 }
Ejemplo n.º 15
0
 public void JobsRunCancel(long runId)
 {
     using (var client = DatabricksClient.CreateClient(this.baseUrl, this.token))
     {
         client.Jobs.RunsCancel(runId).Wait();
     }
 }
        private static async Task GroupsApi(DatabricksClient client)
        {
            Console.WriteLine("Listing groups");
            var groupsList = await client.Groups.List();

            foreach (var group in groupsList)
            {
                Console.WriteLine("Group name: {0}", group);
            }

            const string newGroupName = "sample group";

            Console.WriteLine("Creating new group \"{0}\"", newGroupName);
            await client.Groups.Create(newGroupName);

            Console.WriteLine("Deleting group \"{0}\"", newGroupName);
            await client.Groups.Delete(newGroupName);

            Console.WriteLine("Listing members in admins group");
            var members = await client.Groups.ListMembers("admins");

            foreach (var member in members)
            {
                if (!string.IsNullOrEmpty(member.UserName))
                {
                    Console.WriteLine("Member (User): {0}", member.UserName);
                }
                else
                {
                    Console.WriteLine("Member (Group): {0}", member.GroupName);
                }
            }
        }
Ejemplo n.º 17
0
        /// <summary>
        /// Initializes a new instance of hte <see cref="DatabricksInfoProvider"/> class.
        /// </summary>
        /// <param name="databricksClient">The client to interact with Databricks.</param>
        /// <param name="logger">The instance to log metric reports of job runs.</param>
        /// <exception cref="ArgumentNullException">Thrown when the <paramref name="databricksClient"/> or <paramref name="logger"/> is <c>null</c>.</exception>
        public DatabricksInfoProvider(DatabricksClient databricksClient, ILogger logger)
        {
            Guard.NotNull(databricksClient, nameof(databricksClient));
            Guard.NotNull(logger, nameof(logger));

            _databricksClient = databricksClient;
            _logger           = logger;
        }
Ejemplo n.º 18
0
        public async Task GetJobInfo()
        {
            using (var client = DatabricksClient.CreateClient(Settings.Instance.Databricks.BaseUrl, Settings.Instance.Databricks.Token))
            {
                var jobId = await client.Jobs.Get(51);

                var run = await client.Jobs.RunsGet(22);
            }
        }
Ejemplo n.º 19
0
        public async Task GetFinishedJobRuns_WithEndTimeLessThanStartTime_Throws()
        {
            // Arrange
            DateTimeOffset   startWindow = BogusGenerator.Date.SoonOffset();
            DateTimeOffset   endWindow   = BogusGenerator.Date.RecentOffset();
            DatabricksClient client      = DatabricksClientFactory.Create(Enumerable.Empty <Run>(), Enumerable.Empty <Job>());
            var provider = new DatabricksInfoProvider(client, NullLogger.Instance);

            // Act / Assert
            await Assert.ThrowsAnyAsync <ArgumentException>(() => provider.GetFinishedJobRunsAsync(startWindow, endWindow));
        }
Ejemplo n.º 20
0
        private static async Task WaitUntilDatabricksJobRunIsCompleted(DatabricksClient client, int jobId)
        {
            AsyncRetryPolicy <RunList> retryPolicy =
                Policy.HandleResult <RunList>(list => list.Runs.Any(r => !r.IsCompleted))
                .WaitAndRetryForeverAsync(index => TimeSpan.FromSeconds(10));

            await Policy.TimeoutAsync(TimeSpan.FromMinutes(7))
            .WrapAsync(retryPolicy)
            .ExecuteAsync(async() => await client.Jobs.RunsList(jobId, activeOnly: true));

            await Task.Delay(TimeSpan.FromMinutes(2));
        }
Ejemplo n.º 21
0
        public SjaasWorker()
        {
            var docDbUri          = "to-be-filled";
            var docDbPrimaryKey   = "to-be-filled";
            var docDbClient       = new DocumentClient(new Uri(docDbUri), docDbPrimaryKey);
            var database          = "Sjaas";
            var jobCollection     = "Job";
            var clusterCollection = "Cluster";
            var sjaasDataStore    = new CosmosDataStore(docDbClient, database, jobCollection, clusterCollection);
            var databricksClient  = DatabricksClient.CreateClient("to-be-filled", "to-be-filled");

            this.jobEventProcessor = new JobEventProcessor(sjaasDataStore, databricksClient);
        }
        /// <summary>
        /// This method is called when the <see cref="T:Microsoft.Extensions.Hosting.IHostedService" /> starts. The implementation should return a task that represents
        /// the lifetime of the long running operation(s) being performed.
        /// </summary>
        /// <param name="stoppingToken">
        ///     Triggered when <see cref="M:Microsoft.Extensions.Hosting.IHostedService.StopAsync(System.Threading.CancellationToken)" /> is called.
        /// </param>
        /// <returns>
        ///     A <see cref="T:System.Threading.Tasks.Task" /> that represents the long running operations.
        /// </returns>
        public async Task ExecuteAsync(CancellationToken stoppingToken)
        {
            using (DatabricksClient client = await _options.CreateDatabricksClientAsync(_secretProvider))
                using (var databricksInfoProvider = new DatabricksInfoProvider(client, _logger))
                {
                    (DateTimeOffset start, DateTimeOffset end) = _options.DetermineNextTimeWindow();

                    _logger.LogInformation(
                        "Job monitor for Databricks is starting at {TriggerTime} for time windows {WindowStart} - {WindowEnd}",
                        DateTimeOffset.UtcNow, start, end);

                    string metricName = _options.UserOptions.MetricName;
                    await databricksInfoProvider.MeasureJobOutcomesAsync(metricName, start, end);
                }
        }
Ejemplo n.º 23
0
        public async Task GetFinishedJobRuns_WithNoAvailableFinishedJobs_ReturnsNoFinishedJobs()
        {
            // Arrange
            DateTimeOffset startWindow = BogusGenerator.Date.RecentOffset();
            DateTimeOffset endWindow   = BogusGenerator.Date.SoonOffset();

            DatabricksClient client = DatabricksClientFactory.Create(Enumerable.Empty <Run>(), Enumerable.Empty <Job>());
            var provider            = new DatabricksInfoProvider(client, NullLogger.Instance);

            // Act
            IEnumerable <JobRun> finishedJobs = await provider.GetFinishedJobRunsAsync(startWindow, endWindow);

            // Assert
            Assert.Empty(finishedJobs);
        }
Ejemplo n.º 24
0
        public AzureDatabricksDbfsBlobStorage(string baseUri, string token, bool isReadOnly)
        {
            if (baseUri is null)
            {
                throw new ArgumentNullException(nameof(baseUri));
            }
            if (token is null)
            {
                throw new ArgumentNullException(nameof(token));
            }

            _client     = DatabricksClient.CreateClient(baseUri, token);
            _dbfs       = _client.Dbfs;
            _isReadOnly = isReadOnly;
        }
        public async Task Run([TimerTrigger("0 */1 * * * *")] TimerInfo timer, ILogger logger)
        {
            logger.LogInformation($"C# Timer trigger function executed at: {DateTime.UtcNow}");

            var    metricName  = _configuration.GetValue <string>("Arcus:ApplicationInsights:MetricName");
            var    baseUrl     = _configuration.GetValue <string>("Arcus:Databricks:Url");
            string secretToken = await _secretProvider.GetRawSecretAsync("Arcus.Databricks.SecretToken");

            var startOfWindow = timer.ScheduleStatus.Last;
            var endOfWindow   = timer.ScheduleStatus.Next;

            using var client = DatabricksClient.CreateClient(baseUrl, secretToken);
            using (var provider = new DatabricksInfoProvider(client, logger))
            {
                await provider.MeasureJobOutcomesAsync(metricName, startOfWindow, endOfWindow);
            }
        }
Ejemplo n.º 26
0
        public async Task MeasureJobOutcomes_WithNoAvailableFinishedJobs_ReturnsNoFinishedJobs()
        {
            // Arrange
            DateTimeOffset startWindow = BogusGenerator.Date.RecentOffset();
            DateTimeOffset endWindow   = BogusGenerator.Date.SoonOffset();
            string         metricName  = BogusGenerator.Random.Word();

            DatabricksClient client = DatabricksClientFactory.Create(Enumerable.Empty <Run>(), Enumerable.Empty <Job>());
            var spyLogger           = new InMemoryLogger <DatabricksJobMetricsJob>();
            var provider            = new DatabricksInfoProvider(client, spyLogger);

            // Act
            await provider.MeasureJobOutcomesAsync(metricName, startWindow, endWindow);

            // Assert
            Assert.DoesNotContain(spyLogger.Messages, msg => msg.StartsWith("Metric " + metricName));
        }
Ejemplo n.º 27
0
        public DatabricksBlobStorage(string baseUri, string token)
        {
            if (baseUri is null)
            {
                throw new ArgumentNullException(nameof(baseUri));
            }
            if (token is null)
            {
                throw new ArgumentNullException(nameof(token));
            }

            _nativeClient = DatabricksClient.CreateClient(baseUri, token);

            Mount("dbfs", new DbfsStorage(_nativeClient.Dbfs));
            Mount("workspace", new WorkspaceStorage(_nativeClient.Workspace));
            Mount("secrets", _ss = new SecretStorage(_nativeClient.Secrets));
            Mount("jobs", new JobsStorage(_nativeClient.Jobs));
        }
Ejemplo n.º 28
0
        public async Task MeasureJobOutcomes_WithinTimeWindow_OnlyReturnsFinishedJobsWithinTheTimeWindow()
        {
            // Arrange
            DateTimeOffset startWindow = BogusGenerator.Date.RecentOffset();
            DateTimeOffset endWindow   = BogusGenerator.Date.SoonOffset();
            string         metricName  = BogusGenerator.Random.Word();

            IEnumerable <Run> includedRuns = CreateRandomRuns(startWindow, endWindow);
            IEnumerable <Run> tooLateRuns  = CreateRandomRuns(endWindow, BogusGenerator.Date.FutureOffset());
            IEnumerable <Run> allRuns      = includedRuns.Concat(tooLateRuns);
            IEnumerable <Job> jobs         = allRuns.Select(r => new Job
            {
                JobId    = r.JobId,
                Settings = new JobSettings {
                    Name = Guid.NewGuid().ToString()
                }
            }).ToArray();

            DatabricksClient client = DatabricksClientFactory.Create(allRuns, jobs);
            var spyLogger           = new InMemoryLogger <DatabricksJobMetricsJob>();
            var provider            = new DatabricksInfoProvider(client, spyLogger);

            // Act
            await provider.MeasureJobOutcomesAsync(metricName, startWindow, endWindow);

            // Assert
            Assert.All(includedRuns, run =>
            {
                Assert.Contains(spyLogger.Messages, msg => msg.Contains(run.RunId.ToString()));
                Assert.Contains(spyLogger.Messages, msg => msg.Contains(run.JobId.ToString()));
                Job job = Assert.Single(jobs, j => j.JobId == run.JobId);
                Assert.NotNull(job);
                Assert.Contains(spyLogger.Messages, msg => msg.Contains(job.Settings.Name));
            });
            Assert.All(tooLateRuns, run =>
            {
                Assert.DoesNotContain(spyLogger.Messages, msg => msg.Contains(run.RunId.ToString()));
                Assert.DoesNotContain(spyLogger.Messages, msg => msg.Contains(run.JobId.ToString()));
                Job job = Assert.Single(jobs, j => j.JobId == run.JobId);
                Assert.NotNull(job);
                Assert.DoesNotContain(spyLogger.Messages, msg => msg.Contains(job.Settings.Name));
            });
        }
        public async Task FinishedDatabricksJobRun_GetsNoticedByRepeatedlyDatabricksJob_ReportsAsMetric()
        {
            // Arrange
            string baseUrl = GetDatabricksUrl();
            string token = GetDatabricksToken();
            int jobId = GetDatabricksJobId();

            using (var client = DatabricksClient.CreateClient(baseUrl, token))
            {
                // Act
                await client.Jobs.RunNow(jobId, RunParameters.CreateNotebookParams(Enumerable.Empty<KeyValuePair<string, string>>()));

                // Assert
                RetryAssertion(
                    () => Assert.Contains(_spyLogger.Messages, msg => msg.StartsWith("Metric Databricks Job Completed")),
                    timeout: TimeSpan.FromMinutes(3),
                    interval: TimeSpan.FromSeconds(1));
            }
        }
Ejemplo n.º 30
0
        private static async Task LibrariesApi(DatabricksClient client)
        {
            Console.WriteLine("All cluster statuses");
            var libraries = await client.Libraries.AllClusterStatuses();

            foreach (var(clusterId, libraryStatuses) in libraries)
            {
                Console.WriteLine("Cluster: {0}", clusterId);

                foreach (var status in libraryStatuses)
                {
                    Console.WriteLine("\t{0}\t{1}", status.Status, status.Library);
                }
            }

            const string testClusterId = "0530-210517-viced348";

            Console.WriteLine("Getting cluster statuses for {0}", testClusterId);
            var statuses = await client.Libraries.ClusterStatus(testClusterId);

            foreach (var status in statuses)
            {
                Console.WriteLine("\t{0}\t{1}", status.Status, status.Library);
            }

            var mvnlibraryToInstall = new MavenLibrary
            {
                MavenLibrarySpec = new MavenLibrarySpec
                {
                    Coordinates = "org.jsoup:jsoup:1.7.2",
                    Exclusions  = new[] { "slf4j:slf4j" }
                }
            };

            await TestInstallUninstallLibrary(client, mvnlibraryToInstall, testClusterId);

            var whlLibraryToInstall = new WheelLibrary
            {
                Wheel = "dbfs:/mnt/dbfsmount1/temp/docutils-0.14-py3-none-any.whl"
            };

            await TestInstallUninstallLibrary(client, whlLibraryToInstall, testClusterId);
        }