コード例 #1
0
        private string GetFtpFolder(NodeConfig nodeConfig, Resource resource, CopyPhase phase)
        {
            string ftpFolder           = resource.Nodes.First(n => n.NodeName == nodeConfig.NodeName).DataFolders.ExchangeUrlFromSystem;
            string incarnatedFtpFolder = IncarnationParams.IncarnatePath(ftpFolder, TaskId, phase);

            return(incarnatedFtpFolder);
        }
コード例 #2
0
ファイル: Model.cs プロジェクト: kbochenina/Kraken
        public HistorySample(
            string package, string resourceName, NodeConfig[] nodesConfig, 
            Dictionary<string, string> packParams, Dictionary<string, double> modelCoefs,
            TimeSpan calcTime, PackageEngine estimatorEngine)
        {
            Package = package;
            ResourceName = resourceName;
            NodesConfig = nodesConfig;

            PackParams = packParams;
            ModelCoefs = modelCoefs;

            CalcTime = calcTime;

            EstimatorEngine = estimatorEngine;
        }
コード例 #3
0
ファイル: TaskSchedule.cs プロジェクト: kbochenina/Kraken
 private TaskSchedule()
 {
     Nodes = new NodeConfig[0];
     FailReason = null;
     ModifiedParams = new Dictionary<string, string>();
 }
コード例 #4
0
ファイル: TaskSchedule.cs プロジェクト: kbochenina/Kraken
        public static IEnumerable<TaskSchedule> Build(
            IEnumerable<Task> tasks, IEnumerable<Resource> resources, IEnumerable<TaskDependency> dependencies,
            out Dictionary<ulong, Dictionary<NodeConfig, Estimation>> estimations)
        {
            estimations = new Dictionary<ulong, Dictionary<NodeConfig, Estimation>>();

            if (tasks == null || !tasks.Any())
            {
                Log.Warn("No tasks to schedule");
                return Enumerable.Empty<TaskSchedule>();
            }

            if (resources == null || !resources.Any())
            {
                Log.Warn("No resources to allocate");
                return Enumerable.Empty<TaskSchedule>();
            }

            var schedule = new List<TaskSchedule>();

            Log.Info(String.Format("Checking permissions for tasks [{0}]", String.Join(", ", tasks.Select(t => t.TaskId.ToString()))));
            var failReasonForTask = new Dictionary<ulong, string>();
            var permissionsForTask = GetPermissionsForTasks(tasks, resources, out failReasonForTask);

            foreach (var task in tasks)
            {
                // todo : move to SimpleAllocator
                if (/*task.State == TaskState.ReadyToExecute &&*/ !permissionsForTask[task.TaskId].Any())
                {
                    if (!failReasonForTask.ContainsKey(task.TaskId))
                    {
                        Log.Warn("Unknown permission error");
                        failReasonForTask[task.TaskId] = "Can't run task bacause of unknown permission error";
                    }

                    schedule.Add(new TaskSchedule()
                    {
                        TaskId = task.TaskId,
                        Action = ScheduledAction.Fail,
                        FailReason = failReasonForTask[task.TaskId],
                        Nodes = new NodeConfig[0],
                    });
                }
            }

            Log.Debug(
                "Permissions for tasks are: " +
                String.Join("; ", tasks.Select(t =>
                    String.Format("{0} -> {1}", t.TaskId, String.Join(", ", permissionsForTask[t.TaskId]))
                ))
            );

            // Get Estimations
            var estimationsForTask = new Dictionary<ulong, Dictionary<NodeConfig, Estimation>>();
            foreach (var task in tasks)
            {
                task.Time.AddToOverheads(TaskTimeOverheads.Estimation, () =>
                {
                    var permittedNodes = resources.SelectMany(r => r.Nodes).Where(n =>
                            permissionsForTask[task.TaskId].Contains(n.ResourceName + "." + n.NodeName)
                        );

                    // todo : use CoresAvailable if can run now, and CoresTotal if in the future
                    estimationsForTask[task.TaskId] = PackageBaseProxy.GetEstimationsByModel(task.PackageEngineState, resources, permittedNodes);

                    foreach (var node in permittedNodes)
                    {
                        if (!estimationsForTask[task.TaskId].Keys.Any(config => config.NodeName == node.NodeName))
                        {
                            // estimate from history

                            var config = new NodeConfig
                            {
                                NodeName = node.NodeName,
                                ResourceName = node.ResourceName,
                                Cores = 1,
                            };

                            // todo: [!] estimate from history
                            estimationsForTask[task.TaskId][config] = new Estimation(
                                null,
                                null //new HistoryEstimation(1337 + node.NodeName.Length)
                            );

                            estimationsForTask[task.TaskId][config].CalcDuration = TimeSpan.FromSeconds(1.337 + node.NodeName.Length);
                        }
                    }
                });
            }

            try
            {
                estimations = estimationsForTask;

                Log.Debug(
                    "Estimations for tasks are: " +
                    String.Join("; ", tasks.Select(t =>
                        String.Format("{0} -> {1}",
                            t.TaskId,
                            String.Join(", ", estimationsForTask[t.TaskId]
                                .Select(pair => String.Format("{0} on {1}.{2}", pair.Value.CalcDuration, pair.Key.ResourceName, pair.Key.NodeName))
                            )
                        )
                    ))
                );
            }
            catch (Exception estimEx)
            {
                Log.Warn(estimEx.ToString());
            }

            bool scheduledSuccessfully = false;
            try
            {
                Log.Info("Scheduling");
                var schedulingStrated = DateTime.Now;

                var scheduleByScheduler = Reschedule(tasks, resources, dependencies, /*permissionsForTask,*/ estimationsForTask);

                var schedulingFinished = DateTime.Now;
                var schedulingTime = schedulingFinished - schedulingStrated;
                //                var schedulingTimeOnOneTask = TimeSpan.FromMilliseconds(schedulingTime.TotalMilliseconds / scheduleByScheduler.Count());
                Log.Debug(String.Format("Scheduling took {0} seconds", schedulingTime.TotalSeconds));

                foreach (var schedResult in scheduleByScheduler)
                {
                    tasks.Single(t => t.TaskId == schedResult.TaskId)
                        //.Time.AddToOverheads(TaskTimeOverheads.Scheduler, schedulingTimeOnOneTask);
                        .Time.AddToOverheads(TaskTimeOverheads.Scheduler, schedulingTime);
                }

                if (scheduleByScheduler != null && scheduleByScheduler.Any())
                {
                    var nonPermittedSchedule = scheduleByScheduler.FirstOrDefault(s =>
                        s.Action == ScheduledAction.Run &&
                        s.Nodes.Select(n => n.ResourceName + "." + n.NodeName).Except(permissionsForTask[s.TaskId]).Any());

                    if (nonPermittedSchedule != null)
                    {
                        // scheduled to run on non-permitted resource
                        Log.Error(String.Format(
                            "Scheduler scheduled task {0} on non-permitted resource config: '{1}', nodes '{2}'",
                            nonPermittedSchedule.TaskId, nonPermittedSchedule.ResourceName,
                            String.Join("', '", nonPermittedSchedule.Nodes.Select(n => n.NodeName))
                        ));
                        scheduledSuccessfully = false;
                    }
                    else
                    {
                        var busyNodes = resources.SelectMany(r => r.Nodes.Where(n =>
                            (n.CoresAvailable <= 0 || n.SubmissionsAvailable <= 0) && // n.CoresCount > 0 &&
                            !scheduleByScheduler.Any(s => s.Action == ScheduledAction.Abort && s.Nodes.Any(sn => sn.NodeName == n.NodeName && sn.ResourceName == n.ResourceName))
                        )).Select(n => n.ResourceName + "." + n.NodeName);

                        var nodeLimits = resources.SelectMany(r => r.Nodes.Select(n =>
                            new
                            {
                                ResourceName = n.ResourceName,
                                NodeName = n.NodeName,
                                CoresLimit = n.CoresAvailable,
                                TasksLimit = n.TasksSubmissionLimit
                            }
                        )).ToDictionary(lim => lim.ResourceName + "." + lim.NodeName, lim => lim);

                        var nodeAborts = scheduleByScheduler
                            .Where(s => s.Action == ScheduledAction.Abort)
                            .SelectMany(s => s.Nodes)
                            .GroupBy(s => s.ResourceName + "." + s.NodeName)
                            .ToDictionary(group => group.Key, group => new { Tasks = group.Count(), Cores = group.Sum(s => s.Cores) });

                        var overtargetedNodes = scheduleByScheduler
                            .Where(s => s.Action == ScheduledAction.Run)
                            .SelectMany(s => s.Nodes)
                            .GroupBy(n => n.ResourceName + "." + n.NodeName)
                            .Where(
                                group => group.Count() > nodeLimits[group.Key].TasksLimit + (nodeAborts.ContainsKey(group.Key) ? nodeAborts[group.Key].Tasks : 0) ||
                                group.Sum(n => n.Cores) > nodeLimits[group.Key].CoresLimit + (nodeAborts.ContainsKey(group.Key) ? nodeAborts[group.Key].Cores : 0))
                            .Select(group => group.Key);

                        // todo: disallow targeting same node twice in schedule for one task
                        var insaneDecisions = scheduleByScheduler.Join(tasks, s => s.TaskId, t => t.TaskId, (s, t) =>
                            ((s.Action == ScheduledAction.Run && t.State != TaskState.ReadyToExecute) ||
                             (s.Action == ScheduledAction.Run && String.IsNullOrEmpty(s.ResourceName)) ||
                             (s.Action == ScheduledAction.Run && !s.Nodes.Any()) ||
                                //(s.Action == ScheduledAction.Run && s.Nodes.Any(n => n.Cores <= 0)) ||
                             (s.Action == ScheduledAction.Run && busyNodes.Intersect(s.Nodes.Select(conf => conf.ResourceName + "." + conf.NodeName)).Any()) ||
                             (s.Action == ScheduledAction.Run && overtargetedNodes.Intersect(s.Nodes.Select(conf => conf.ResourceName + "." + conf.NodeName)).Any()) ||
                             (s.Action == ScheduledAction.Abort && t.State != TaskState.Started))
                            ? new
                            {
                                action = s.Action,
                                state = t.State,
                                taskId = t.TaskId,
                                resourceName = s.ResourceName,
                                nodeNames = s.Nodes.Select(n => n.NodeName)
                            } : null
                        ).Where(r => r != null);

            //                        if (insaneDecisions.Any())
            //                        {
            //                            if (busyNodes.Any())
            //                                Log.Debug("Busy nodes: " + String.Join(", ", busyNodes));
            //
            //                            if (overtargetedNodes.Any())
            //                                Log.Debug("Overtargeted nodes: " + String.Join(", ", overtargetedNodes));
            //
            //                            Log.Error(
            //                                "Scheduler made some insane decisions: " +
            //                                String.Join(", ",
            //                                    insaneDecisions.Select(d => String.Format(
            //                                        "{0} {1} task '{2}' on {3}({4})",
            //                                        d.action, d.state, d.taskId,
            //                                        d.resourceName, String.Join(", ", d.nodeNames)
            //                                    ))
            //                                ) +
            //                                ". Ignoring them."
            //                            );
            //
            //                            //Log.Warn("NOT ignoring insane schedule for debug purposes"); // todo : [!] remove
            //                            scheduleByScheduler = scheduleByScheduler.Where(s => !insaneDecisions.Any(d => d.taskId == s.TaskId)).ToArray();
            //                        }

                        if (scheduleByScheduler.Any())
                        {
                            schedule.AddRange(scheduleByScheduler);
                            scheduledSuccessfully = true;
                        }
                        else
                        {
                            scheduledSuccessfully = false;
                        }
                    }
                }
                else if (tasks.All(t => t.State == TaskState.Started))
                {
                    Log.Info("All tasks sento to the scheduler are running. Ignore null or empty schedule.");
                    scheduledSuccessfully = true;
                }
                else
                    Log.Warn("Scheduler returned bad schedule (either null or empty).");
            }
            catch (Exception e)
            {
                Log.Error(String.Format(
                    "Exception in scheduler: {0}\n{1}",
                    e.Message, e.StackTrace
                ));
            }

            //todo: remove it later
            //            scheduledSuccessfully = false;

            if (!scheduledSuccessfully)
            {
                try
                {
                    Log.Info("Using simple task allocator");
                    var scheduleByAllocator = AllocateTasks(tasks, resources, /* permissionsForTask, */ estimationsForTask);
                    schedule.AddRange(scheduleByAllocator);
                }
                catch (Exception e)
                {
                    Log.Error(String.Format(
                        "Exception in allocator: {0}\n{1}",
                        e.Message, e.StackTrace
                    ));
                }
            }

            return schedule;
        }
コード例 #5
0
ファイル: Task.cs プロジェクト: kbochenina/Kraken
 private string GetFtpFolder(NodeConfig nodeConfig, Resource resource, CopyPhase phase)
 {
     string ftpFolder = resource.Nodes.First(n => n.NodeName == nodeConfig.NodeName).DataFolders.ExchangeUrlFromSystem;
     string incarnatedFtpFolder = IncarnationParams.IncarnatePath(ftpFolder, TaskId, phase);
     return incarnatedFtpFolder;
 }