示例#1
0
        static void Main(string[] args)
        {
            string databricksToken = GetSecret("ngrmdfvtkvdev", "DatabricksToken");
            string databricksdeviationsCalculationJobName = "BatchedDeviations_2_0_1";
            int    surveyId = 28;

            ILogger log = new ConsoleLogger();

            log.LogInformation($"SurveyID: {surveyId}");

            Databricks dbUtil = new Databricks(databricksToken, "northeurope.azuredatabricks.net", log);
            JobList    dbJobs = dbUtil.ListJobs();
            int?       jobId  = dbUtil.GetJobIdByName(dbJobs, databricksdeviationsCalculationJobName);

            int?runId = null;

            for (int i = 0; i < 199; i++)
            {
                var deviationRuns = dbUtil.GetActiveRuns(jobId);
                var surveyRuns    = GetSurveyRuns(surveyId, deviationRuns, log);
                if (surveyRuns.Count == 0)
                {
                    if (runId != null) // previous run exists and is completed
                    {
                        string  output = dbUtil.GetRunNotebookOutput(runId);
                        JObject o      = JObject.Parse(output);
                        string  result = o.SelectToken("notebook_output.result").ToString();
                        log.LogInformation(result);
                        if (result == "No sequences to calculate. Exiting." || result == "Nothing to process. Exiting.")
                        {
                            break;
                        }
                    }

                    log.LogInformation($"{i}: No runs for SurveyID: {surveyId}. Starting.");

                    var notebookParams = new Dictionary <string, dynamic> {
                        { "survey_id", surveyId }
                    };
                    RunNowJob      jobRun   = new RunNowJob((int)jobId, notebookParams);
                    RunNowResponse response = dbUtil.RunJob(jobRun);


                    runId = response.RunId;
                }
                else
                {
                    log.LogInformation($"{i}: {surveyRuns.Count} run(s) for SurveyID: {surveyId} in progress. Skipping.");
                }

                Thread.Sleep(5000); // the NGRMDFVT_DeviationsCalculationRunner kicks off every 5 seconds
            }
        }
示例#2
0
        public RunNowResponse RunJob(RunNowJob job)
        {
            RestRequest request = new RestRequest("2.0/jobs/run-now");

            request.AddHeader("Authorization", $"Bearer {authorizationToken}");
            request.AddJsonBody(job);

            IRestResponse <RunNowResponse> response = restClient.Post <RunNowResponse>(request);

            if (response.StatusCode != System.Net.HttpStatusCode.OK)
            {
                throw new ApplicationException($"({response.StatusCode}) Could not start a new run: {response.Content} ");
            }

            return(response.Data);
        }