public Scheduler.Estimated.LaunchPlan RescheduleEstimated(Scheduler.Estimated.EstimatedWorkflow workflow)
{
logger.Debug(string.Format(debugPrefixNameString, "RescheduleEstimated"));
try
{
var wfs = ConstructWorkflows(workflow).ToList();
DeleteFinishedWorkflows(ref wfs);
if (scheduledWfCount == 0)
{
//firstCall = DateTime.Now;
ReadWindows();
PrintActiveToLog(wfs);
PrintEstimatedToLog(wfs);
// wfId, (taskId1, nodeId1, startTime), ..., (taskIdN, nodeIdN, startTime) for all WFs to be scheduled
schedule = new Dictionary<int, List<Tuple<int, int, int>>>();
// nodeId, (wfId, taskId, startTime according to schedule)
nodeQueues = new Dictionary<int, List<Tuple<int, int, int>>>();
// (wf.Item1, wfId)
wfIDs = new Dictionary<string, int>();
// (nodeId, (wfId, currentTaskId)) - we suppose that one node can execute only one task at time!!!
nodeCurrent = new Dictionary<int, Tuple<int, int>>();
// read number of nodes from resFile
int nodesCount = ReadNodesCount();
if (nodesCount == -1)
logger.Info("ReadNodesCount(). Wrong nodes count");
for (int i = 1; i <= nodesCount; i++)
{
nodeCurrent.Add(i, null);
nodeQueues.Add(i, new List<Tuple<int, int, int>>());
}
// (wfId, taskId)
// (wfId, taskId)
taskIDs = new Dictionary<ulong, Tuple<int, int>>();
// clear from previous runs
_scheduledWfs.Clear();
wasActive = new Dictionary<ulong, bool>();
foreach (var wf in wfs)
{
if (!tasksCount.ContainsKey(wf.Item1))
tasksCount.Add(wf.Item1, wf.Item3.Count());
foreach (var task in wf.Item3)
{
wasActive.Add(task.Id, false);
}
}
}
PrintActiveToLog(wfs);
SetActive(ref wfs);
//PrintEstimatedToLog(wfs);
var result = new LaunchPlan();
if (ConfigurationManager.AppSettings["SchedMethod"] == "second"){
//DeleteFinishedTasksFromQueues(ref wfs);
//logger.Info("DeleteFinishedTasksFromQueues() ended");
DeleteFinishedTasksFromQueues(ref wfs);
logger.Info("DeleteFinishedTasksFromQueues() ended");
DeleteFinishedWorkflows(ref wfs);
logger.Info("DeleteFinishedWorkflows() ended");
//logger.Info("RescheduleEstimated(). Keys of scheduled workflows: ");
//foreach (var wf in _scheduledWfs)
//{
// logger.Info("Workflow " + wf.Key);
//}
// (wfKey, wfDep, wfActiveEstimated)
var wfResults = new Dictionary<string, Tuple<IEnumerable<Scheduler.Estimated.TasksDepenendency>, List<ActiveEstimatedTask>>>();
//logger.Info("Wf ids: ");
foreach (var wf in wfs)
{
// if workflow was already scheduled, copy previous schedule
if (_scheduledWfs.ContainsKey(wf.Item1))
{
wfResults.Add(wf.Item1, new Tuple<IEnumerable<Scheduler.Estimated.TasksDepenendency>, List<ActiveEstimatedTask>>(wf.Item4, _scheduledWfs[wf.Item1].Plan));
}
}
//logger.Info("Scheduled WFs count: " + _scheduledWfs.Count());
//logger.Info("Current plan after already scheduled wf's addition");
//PrintLaunchPlanToLog(result);
var tosched = wfs.Where(wf => !_scheduledWfs.ContainsKey(wf.Item1)).ToArray();
if (tosched.Count() != 0)
{
// wf ids scheduled on current step
List<int> schedWFIds = new List<int>();
int idToAdd = 0;
if (wfIDs.Count == 0)
idToAdd = 0;
else idToAdd = wfIDs.ElementAt(wfIDs.Count - 1).Value;
foreach (var wf in tosched)
{
wfIDs.Add(wf.Item1, idToAdd);
schedWFIds.Add(idToAdd);
foreach (var task in wf.Item3)
{
taskIDs.Add(task.Id, new Tuple<int, int>(idToAdd, Int32.Parse(task.Parameters["id"])));
}
idToAdd++;
}
// removing old *.dat files if they exist
// generating *.dat files for each unscheduled wf
for (int i = 0; i < tosched.Length; i++)
{
//logger.Info("To sched key: {0}", tosched[i].Item1);
GenerateWorkflowInfo(ref tosched[i], wfIDs[tosched[i].Item1], i);
}
try
{
Process proc = new Process();
proc.StartInfo.UseShellExecute = false;
proc.StartInfo.RedirectStandardOutput = true;
proc.StartInfo.RedirectStandardError = true;
string fullPathToScheduler = ConfigurationManager.AppSettings["SchedPath"] + "\\WFSched.exe";
logger.Info("Starting " + fullPathToScheduler + "...");
proc.StartInfo.FileName = fullPathToScheduler;
proc.Start();
string output = proc.StandardOutput.ReadToEnd();
string error = proc.StandardError.ReadToEnd();
proc.WaitForExit();
var exitCode = proc.ExitCode;
logger.Info("Scheduler was exit with code " + exitCode.ToString());
//logger.Info("Output: " + output);
//logger.Info("Error: " + error);
}
catch (Exception e)
{
logger.Info("WFSched exception: " + e.Message);
}
ReadScheduleFromFile(ref schedule);
// add information to resource queues
AddSchedWfToQueues(ref schedule, ref schedWFIds);
foreach (var wf in tosched)
{
var tasks = wf.Item3;
var wfID = wfIDs[wf.Item1];
ResourceEstimation[] estimations = null;
estimations = new ResourceEstimation[tasks.Count()];
foreach (var task in tasks)
{
ResourceEstimation estimation = new ResourceEstimation();
var localTaskID = int.Parse(task.Parameters["id"]) - 1;
//logger.Info("Task WFID:", localTaskID);
var taskNode = schedule[wfID].Where(t => t.Item1 == localTaskID);
//logger.Info("Before: wfID " + wfID + " localTaskID: " + localTaskID);
var nodeIndex = taskNode.First().Item2;
//logger.Info("After");
try
{
estimation = task.Estimations[nodeIndex];
estimations[localTaskID] = estimation;
}
catch (Exception ex)
{
logger.ErrorException("Exception in getting estimations", ex);
}
scheduledWfCount++;
}
try
{
var activeestimated = processUnscheduledWfsSeq(wf, estimations);
wfResults.Add(wf.Item1, new Tuple<IEnumerable<Scheduler.Estimated.TasksDepenendency>, List<ActiveEstimatedTask>>(wf.Item4, activeestimated));
}
catch (Exception ex)
{
logger.ErrorException("Exception in processUnscheduledWFsSeq", ex);
}
}
}
// update status
UpdateTaskStatus(ref wfResults);
SetStatusToFinishedTasks(ref wfResults);
try
{
foreach (var wf in wfResults)
{
var wfResult = new LaunchPlan();
wfResult.Plan.AddRange(wf.Value.Item2);
result.Plan.AddRange(wf.Value.Item2);
if (_scheduledWfs.ContainsKey(wf.Key))
{
_scheduledWfs[wf.Key] = wfResult;
}
else
{
_scheduledWfs.Add(wf.Key, wfResult);
}
}
}
catch (Exception ex)
{
logger.ErrorException("Add final plan error. " + ex.Message, ex);
}
}
if (isFirstActive)
logger.Info("Current time: " + GetTime());
//logger.Info("Current plan: " + _scheduledWfs.Count());
PrintLaunchPlanToLog(result);
//logger.Info("Workflows tasks count");
foreach (var wf in wfs)
{
logger.Info(wf.Item1 + " active: " + wf.Item2.Count() + " estimated: " + wf.Item3.Count());
}
//prevCall = DateTime.Now;
if (!isFirstActive)
CheckFirstActive(ref wfs);
return result;
}
catch (Exception ex)
{
logger.ErrorException("Exception in scheduling. " + ex.Message, ex);
throw;
}
}
private List<ActiveEstimatedTask> processUnscheduledWfsSeq(Tuple<string, IEnumerable<ActiveEstimatedTask>, IEnumerable<EstimatedTask>, IEnumerable<TasksDepenendency>> wf, ResourceEstimation[] est)
{
var result = new List<ActiveEstimatedTask>();
// foreach (var wf in wfs)
//{
// Should be empty, i.e. we mustn't get here if we have a wf with active tasks
var runningOrScheduledTasks = wf.Item2;
var notScheduledTasks = wf.Item3;
var dependencies = wf.Item4;
List<ActiveEstimatedTask> schedTasks = null;
schedTasks = new List<ActiveEstimatedTask>();
int localTaskID = 0;
try
{
foreach (var task in notScheduledTasks)
{
localTaskID = Convert.ToInt32(task.Parameters["id"]) - 1;
//logger.Info("Local task id: " + localTaskID);
//logger.Info(" NodeName: " + est[localTaskID].Resource.Nodes[0].DNSName);
ActiveEstimatedTask taskToAdd = new ActiveEstimatedTask();
taskToAdd.ApplicationName = task.ApplicationName;
taskToAdd.WFid = task.WFid;
taskToAdd.Id = task.Id;
taskToAdd.IsUrgent = false;
taskToAdd.Parameters = new Dictionary<string, string>(task.Parameters);
//This state must be the same for all tasks. Correct conversion will be done in ExecutionBroker
//taskToAdd.State = TaskScheduler.TaskState.LAUNCHED;
taskToAdd.State = TaskScheduler.TaskState.SCHEDULED;
taskToAdd.Estimation = new ActiveEstimation()
{
Destination = new LaunchDestination()
{
ResourceName = est[localTaskID].Resource.Name,
NodeNames = new string[] { est[localTaskID].Resource.Nodes[0].DNSName }
},
Resource = new Resource()
{
Name = est[localTaskID].Resource.Name,
Nodes = new Node[] { est[localTaskID].Resource.Nodes[0] },
Parameters = task.Parameters,
},
LaunchTime = est[localTaskID].Result.CalculationTime,
Result = est[localTaskID].Result
};
schedTasks.Add(taskToAdd);
}
result.AddRange(schedTasks);
//logger.Info("Scheduled tasks count: " + schedTasks.Count);
}
catch (Exception ex)
{
logger.ErrorException("Exception in processUnscheduledWfsSeq", ex);
}
return result;
//var schedTasks = notScheduledTasks.Select(task => new ActiveEstimatedTask()
//{
// ApplicationName = task.ApplicationName,
// WFid = task.WFid,
// Id = task.Id,
// IsUrgent = false,
// Parameters = new Dictionary<string, string>(task.Parameters),
// //This state must be the same for all tasks. Correct conversion will be done in ExecutionBroker
// State = TaskScheduler.TaskState.LAUNCHED,
// Estimation = new ActiveEstimation()
// {
// Destination = new LaunchDestination()
// {
// ResourceName = est[localTaskID].Resource.Name,
// NodeNames = new string[] { est[localTaskID].Resource.Nodes[0].DNSName },
// },
// Resource = new Resource()
// {
// Name = est[localTaskID].Resource.Name,
// Nodes = new Node[] { est[localTaskID].Resource.Nodes[0] },
// Parameters = task.Parameters,
// },
// LaunchTime = est[localTaskID].Result.CalculationTime,
// Result = est[localTaskID].Result,
// },
//}).ToList();
}
