static void Main(string[] args)
{
string databricksToken = GetSecret("ngrmdfvtkvdev", "DatabricksToken");
string databricksdeviationsCalculationJobName = "BatchedDeviations_2_0_1";
int surveyId = 28;
ILogger log = new ConsoleLogger();
log.LogInformation($"SurveyID: {surveyId}");
Databricks dbUtil = new Databricks(databricksToken, "northeurope.azuredatabricks.net", log);
JobList dbJobs = dbUtil.ListJobs();
int? jobId = dbUtil.GetJobIdByName(dbJobs, databricksdeviationsCalculationJobName);
int?runId = null;
for (int i = 0; i < 199; i++)
{
var deviationRuns = dbUtil.GetActiveRuns(jobId);
var surveyRuns = GetSurveyRuns(surveyId, deviationRuns, log);
if (surveyRuns.Count == 0)
{
if (runId != null) // previous run exists and is completed
{
string output = dbUtil.GetRunNotebookOutput(runId);
JObject o = JObject.Parse(output);
string result = o.SelectToken("notebook_output.result").ToString();
log.LogInformation(result);
if (result == "No sequences to calculate. Exiting." || result == "Nothing to process. Exiting.")
{
break;
}
}
log.LogInformation($"{i}: No runs for SurveyID: {surveyId}. Starting.");
var notebookParams = new Dictionary <string, dynamic> {
{ "survey_id", surveyId }
};
RunNowJob jobRun = new RunNowJob((int)jobId, notebookParams);
RunNowResponse response = dbUtil.RunJob(jobRun);
runId = response.RunId;
}
else
{
log.LogInformation($"{i}: {surveyRuns.Count} run(s) for SurveyID: {surveyId} in progress. Skipping.");
}
Thread.Sleep(5000); // the NGRMDFVT_DeviationsCalculationRunner kicks off every 5 seconds
}
}
public RunNowResponse RunJob(RunNowJob job)
{
RestRequest request = new RestRequest("2.0/jobs/run-now");
request.AddHeader("Authorization", $"Bearer {authorizationToken}");
request.AddJsonBody(job);
IRestResponse <RunNowResponse> response = restClient.Post <RunNowResponse>(request);
if (response.StatusCode != System.Net.HttpStatusCode.OK)
{
throw new ApplicationException($"({response.StatusCode}) Could not start a new run: {response.Content} ");
}
return(response.Data);
}