void startListenerAndMonitor()
{
if (!startListener())
{
throw new ApplicationException("Unable to start orchestrator listener! Unable to continue...");
}
_report.addInfo("The orchestrator service has started!");
while (true)
{
try
{
_report.addDebug("The main orchestrator process will sleep before checking it's job stack again");
System.Threading.Thread.Sleep(new TimeSpan(0, _cronSchedule, 0));
// first check if any of our workstacks have items - if not, see if db says we have work
if (_workStack.Count() == 0 && _activeExtractions.Count() == 0 && _erroredExtractions.Count() == 0 && _completedExtractions.Count() == 0)
{
getConfigs();
if (_configurations.Count == 0)
{
_report.addDebug("Didn't find any scheduled jobs for this iteration");
continue; // nothing on any of the collections and no jobs reported by getConfigs - SLEEP!
}
else // found configs! let's set the worklist
{
_report.addDebug("Found jobs scheduled for this iteration! Setting worklists and new batch ID");
setBatchIdAndWorklist();
}
}
//System.Threading.Thread.Sleep(sleepUntil.Subtract(curTime).Duration());
//sleepUntil = DateTime.Now.AddMinutes(_cronSchedule);
//_serviceState.NextRun = DateTime.Now.AddMinutes(_cronSchedule);
_report.addDebug(String.Format("The orchestrator service has completed an iteration - one configured every {0} minutes", _cronSchedule.ToString()));
_report.addInfo("The orchestrator reported " + _workStack.Count() + " jobs still on the stack at iteration");
_report.addInfo("The orchestrator reported " + _completedExtractions.Count() + " completed jobs at iteration");
_report.addInfo("The orchestrator reported " + _erroredExtractions.Count() + " errored jobs at iteration");
_report.addInfo("The orchestrator reported " + _activeExtractions.Count() + " active jobs at iteration");
if (_activeExtractions.Count() > 0 || _workStack.Count() > 0) // if there is still work occurring, don't clear anything - wait until jobs finish
{
_report.addDebug("Found active extractions/jobs on workstack - going back to sleep");
continue;
}
if (_completedExtractions.Count() > 0 || _erroredExtractions.Count() > 0) // only save our report if something interesting happened
{
_report.addInfo(String.Format("Looks like all jobs finished for batch {0}! - getting ready to save report", _report.BatchId));
_report.EndTimestamp = DateTime.Now;
try
{
_sqlDao.saveReport(_report);
logging.Log.LOG("Successfully saved report to db");
}
catch (Exception exc)
{
logging.Log.LOG("There was a problem saving the report to SQL: " + exc.ToString());
}
}
sleepForDashboardIfTime();
_completedExtractions.RemoveAll();
_erroredExtractions.RemoveAll();
setBatchIdAndWorklist();
//if (_workStack.Count() == 0) // if we didn't find any jobs to add for this iteration, just continue so we sleep again
//{
// _report.addDebug("Didn't find any jobs to add for this iteration");
// continue;
//}
// TBD - do we really want to do this?
// ensure server is still listening - will throw an exception and be caught below if failure occurs
//Client c = new Client(_myIp.ToString(), _listeningPort);
//ThreadUtils.retryAction(new Action(() => c.connect()), 3, new TimeSpan(0, 1, 0), new ApplicationException("Unable to re-start listener after 10 attempts. Unable to continue..."));
//c.disconnect();
//_report.addDebug("Successfully verified the orchestrator service is listening for requests");
}
catch (Exception exc)
{
_server.SocketContainer.Locked = true;
//_sqlThread.Abort();
_report.addError(exc.Message, exc);
((OrchestratorReport)_report).HasError = "T";
_sqlDao.saveReport(_report);
throw exc;
}
}
}
