static void Main(string[] args) { string databricksToken = GetSecret("ngrmdfvtkvdev", "DatabricksToken"); string databricksdeviationsCalculationJobName = "BatchedDeviations_2_0_1"; int surveyId = 28; ILogger log = new ConsoleLogger(); log.LogInformation($"SurveyID: {surveyId}"); Databricks dbUtil = new Databricks(databricksToken, "northeurope.azuredatabricks.net", log); JobList dbJobs = dbUtil.ListJobs(); int? jobId = dbUtil.GetJobIdByName(dbJobs, databricksdeviationsCalculationJobName); int?runId = null; for (int i = 0; i < 199; i++) { var deviationRuns = dbUtil.GetActiveRuns(jobId); var surveyRuns = GetSurveyRuns(surveyId, deviationRuns, log); if (surveyRuns.Count == 0) { if (runId != null) // previous run exists and is completed { string output = dbUtil.GetRunNotebookOutput(runId); JObject o = JObject.Parse(output); string result = o.SelectToken("notebook_output.result").ToString(); log.LogInformation(result); if (result == "No sequences to calculate. Exiting." || result == "Nothing to process. Exiting.") { break; } } log.LogInformation($"{i}: No runs for SurveyID: {surveyId}. Starting."); var notebookParams = new Dictionary <string, dynamic> { { "survey_id", surveyId } }; RunNowJob jobRun = new RunNowJob((int)jobId, notebookParams); RunNowResponse response = dbUtil.RunJob(jobRun); runId = response.RunId; } else { log.LogInformation($"{i}: {surveyRuns.Count} run(s) for SurveyID: {surveyId} in progress. Skipping."); } Thread.Sleep(5000); // the NGRMDFVT_DeviationsCalculationRunner kicks off every 5 seconds } }
public RunNowResponse RunJob(RunNowJob job) { RestRequest request = new RestRequest("2.0/jobs/run-now"); request.AddHeader("Authorization", $"Bearer {authorizationToken}"); request.AddJsonBody(job); IRestResponse <RunNowResponse> response = restClient.Post <RunNowResponse>(request); if (response.StatusCode != System.Net.HttpStatusCode.OK) { throw new ApplicationException($"({response.StatusCode}) Could not start a new run: {response.Content} "); } return(response.Data); }