public TaskSchedule(TaskSchedule other) : this() { if (other != null) { TaskId = other.TaskId; Action = other.Action; var newNodes = new List<NodeConfig>(); foreach (var node in other.Nodes) { var newNode = node; newNodes.Add(newNode); } Nodes = newNodes; FailReason = other.FailReason; if (other.Estimation != null) Estimation = new Estimation(other.Estimation); //EstimatedStartTime = other.EstimatedStartTime; //EstimatedFinishTime = other.EstimatedFinishTime; if (other.ModifiedParams != null) ModifiedParams = new Dictionary<string, string>(other.ModifiedParams); } }
public static IEnumerable<NodeTotals> FromSchedule(TaskSchedule schedule) { if (schedule == null || String.IsNullOrEmpty(schedule.ResourceName) || schedule.Nodes == null || !schedule.Nodes.Any()) return null; IEnumerable<NodeTotals> totals = null; try { var resource = ResourceBase.GetResourceByName(schedule.ResourceName); var res = schedule.Nodes.Select(nodeConf => new NodeTotals() { NodeName = nodeConf.NodeName, NodeAddress = resource.Nodes.First(n => n.NodeName == nodeConf.NodeName).NodeAddress, CoresUsed = (int)nodeConf.Cores, SupportedArchitectures = resource.Nodes.First(n => n.NodeName == nodeConf.NodeName).SupportedArchitectures, }); totals = res.ToArray(); } catch (Exception e) { Log.Warn("Can't make NodeTotals: " + e.ToString()); totals = null; } return totals; }
public Task(Task otherTask) : base(otherTask) { if (otherTask.OutputParams != null) OutputParams = new Dictionary<string, string>(otherTask.OutputParams); if (otherTask.PackageEngineState != null) this.PackageEngineState = (PackageEngineState) otherTask.PackageEngineState.Clone(); Time = new TaskTimeMeasurement(otherTask.Time); Incarnation = new IncarnationParams(otherTask.Incarnation); if (otherTask.Estimations != null) Estimations = new Dictionary<string, double>(otherTask.Estimations); CurrentSchedule = null; if (otherTask.CurrentSchedule != null) { CurrentSchedule = new TaskSchedule(otherTask.CurrentSchedule); // immutable: this.AssignedResource = otherTask.AssignedResource; this.AssignedNodes = otherTask.AssignedNodes; } State = otherTask.State; _inputsProcessed = otherTask._inputsProcessed; _inputsProcessingError = otherTask._inputsProcessingError; _failReason = otherTask._failReason; if (otherTask._lastEvent != null && otherTask._lastEvent.HasValue) _lastEvent = otherTask._lastEvent.Value; }
private static IEnumerable<TaskSchedule> AllocateTasks( IEnumerable<Task> tasks, IEnumerable<Resource> resources, // IDictionary<ulong, IEnumerable<string>> permissionsForTask IDictionary<ulong, Dictionary<NodeConfig, Estimation>> estimationsForTask) { var now = DateTime.Now; double maxCoresCount = resources.Max(r => r.Nodes.Max(n => n.CoresCount)); var availableNodes = resources .SelectMany(r => r.Nodes.Select(n => new { ResourceName = n.ResourceName, NodeName = n.NodeName, CoresLeft = n.CoresAvailable, TasksLeft = n.SubmissionsAvailable, Packs = n.Packages.Select(p => p.Name.ToLowerInvariant()).ToArray() })) .Where(n => n.CoresLeft > 0 && n.TasksLeft > 0) //.Where(n => n.CoresAvailable > 0) //.OrderBy(n => n.Packages.Count + (n.CoresAvailable / maxCoresCount)) .OrderBy(n => n.Packs.Length + (n.CoresLeft / maxCoresCount)) .ToList(); // todo: log availableNodes var scheduledTasks = new List<TaskSchedule>(); foreach (var task in tasks) { if (task.State == TaskState.ReadyToExecute || task.State == TaskState.Scheduled) { var taskSchedule = new TaskSchedule { TaskId = task.TaskId, Action = ScheduledAction.None, Nodes = new NodeConfig[0], Estimation = new Estimation() //EstimatedStartTime = now, //EstimatedFinishTime = now, }; var nodeForTask = availableNodes.FirstOrDefault(n => n.CoresLeft > 0 && n.TasksLeft > 0 && n.Packs.Contains(task.Package.ToLowerInvariant()) && //n.HasPackage(task.Package) && estimationsForTask[task.TaskId].Any(pair => pair.Key.ResourceName == n.ResourceName && pair.Key.NodeName == n.NodeName && pair.Key.Cores <= n.CoresLeft) // permissionsForTask[task.TaskId].Contains(n.ResourceName + "." + n.NodeName, StringComparer.InvariantCultureIgnoreCase) ); if (nodeForTask != null) { taskSchedule.Action = ScheduledAction.Run; var estimation = estimationsForTask[task.TaskId] .First(pair => pair.Key.ResourceName == nodeForTask.ResourceName && pair.Key.NodeName == nodeForTask.NodeName && pair.Key.Cores <= nodeForTask.CoresLeft) ;//.Value; /* taskSchedule.EstimatedStartTime = now; /* taskSchedule.EstimatedFinishTime = now + estimationsForTask[task.TaskId] .Single(pair => pair.Key.ResourceName == nodeForTask.ResourceName && pair.Key.NodeName == nodeForTask.NodeName) .Value; */ taskSchedule.Estimation = estimation.Value; taskSchedule.Estimation.CalcStart = now; /**/ taskSchedule.Nodes = new[] { new NodeConfig { Cores = estimation.Key.Cores, // was "1" NodeName = nodeForTask.NodeName, ResourceName = nodeForTask.ResourceName, } }; scheduledTasks.Add(taskSchedule); availableNodes.RemoveAll(n => n.ResourceName == nodeForTask.ResourceName && n.NodeName == nodeForTask.NodeName ); if (nodeForTask.CoresLeft > taskSchedule.Nodes.Single().Cores && nodeForTask.TasksLeft > 1) { availableNodes.Add(new { ResourceName = nodeForTask.ResourceName, NodeName = nodeForTask.NodeName, CoresLeft = (int)(nodeForTask.CoresLeft - taskSchedule.Nodes.Single().Cores), TasksLeft = (int)(nodeForTask.TasksLeft - estimation.Key.Cores), // was "- 1" Packs = nodeForTask.Packs }); } // todo: log schedule for task } } } return scheduledTasks; /* var limits = new[] { new { pack = "", otherSoft = "", maxCores = 1, maxNodes = 1}, new { pack = "GAMESS", otherSoft = "", maxCores = 1, maxNodes = 1}, new { pack = "ORCA", otherSoft = "windows", maxCores = 1, maxNodes = 1}, new { pack = "ORCA", otherSoft = "", maxCores = 8, maxNodes = 1}, new { pack = "SEMP", otherSoft = "", maxCores = 1, maxNodes = 1}, new { pack = "Plasmon", otherSoft = "", maxCores = 1, maxNodes = 1}, new { pack = "QDLaser", otherSoft = "", maxCores = 1, maxNodes = 1}, new { pack = "JAggregate", otherSoft = "", maxCores = 1, maxNodes = 1}, //new { pack = "Plasmon", otherSoft = "", maxCores = 1, maxNodes = 1}, //new { pack = "QDLaser", otherSoft = "", maxCores = 8, maxNodes = 1}, //new { pack = "JAggregate", otherSoft = "", maxCores = 8, maxNodes = 1}, //new { pack = "NanoFlow", otherSoft = "", maxCores = 16*3, maxNodes = 3}, new { pack = "NanoFlow", otherSoft = "", maxCores = 8, maxNodes = 1}, new { pack = "MD_KMC", otherSoft = "", maxCores = 8, maxNodes = 1}, new { pack = "MD_KMC", otherSoft = "windows", maxCores = 0, maxNodes = 0}, new { pack = "NTDMFT", otherSoft = "", maxCores = 8, maxNodes = 1}, new { pack = "Dalton", otherSoft = "", maxCores = 1, maxNodes = 1}, new { pack = "NAEN", otherSoft = "", maxCores = 1, maxNodes = 1}, new { pack = "Upconversion", otherSoft = "", maxCores = 8, maxNodes = 1}, new { pack = "TestP", otherSoft = "", maxCores = 1, maxNodes = 1}, new { pack = "Belman", otherSoft = "", maxCores = 8, maxNodes = 1}, new { pack = "SWAN", otherSoft = "", maxCores = 1, maxNodes = 1}, }; var scheduledTasks = new List<TaskSchedule>(); foreach (var task in tasks) { var packLimits = limits.Where(l => String.IsNullOrWhiteSpace(l.pack)); if (limits.Any(limit => limit.pack.ToLower() == task.Package.ToLower())) packLimits = limits.Where(limit => limit.pack.ToLower() == task.Package.ToLower()); foreach (var resource in resources) { foreach (var node in resource.Nodes.Where(n => n.HasPackage(task.Package))) { var properPackLimit = packLimits.Max( lim => lim.otherSoft .Split(new char[] { ' ', '\t', '\r', '\n', ',', ';' }, StringSplitOptions.RemoveEmptyEntries) .Intersect(node.OtherSoftware) .Count() ); // Choose number of cores for each node: var coresOnNode = new List<int>(); int coresToDo = properPackLimit.maxCores; int nodesToDo = properPackLimit.maxNodes; foreach (var node in resource.Nodes) { if (coresToDo > 0 && nodesToDo > 0) { int coresOnCurrentNode = Math.Min(node.CoresAvailable, coresToDo); coresOnNode.Add(coresOnCurrentNode); coresToDo -= coresOnCurrentNode; nodesToDo -= (coresOnCurrentNode == 0) ? 0 : 1; } else coresOnNode.Add(0); } int coresFound = coresOnNode.Sum(); if (coresFound == 0) // haven't found anything return null; // Estimate (clusters with more nodes are preferable, so subtract it's nodes.Count from time estimation): TimeSpan time = new TimeSpan((long) ((Math.Round(18000.0 / coresFound) - resource.Nodes.Count + 60)*SEC_IN_TICKS)); Estimation estimation = new Estimation(task.TaskId, resource.ProviderName, resource.ResourceName, coresOnNode.ToArray()) { ExecutionTime = time }; } } } return scheduledTasks; */ }
public static ResourceTotals FromSchedule(TaskSchedule schedule) { if (schedule == null || String.IsNullOrEmpty(schedule.ResourceName)) return null; ResourceTotals totals = null; try { var resource = ResourceBase.GetResourceByName(schedule.ResourceName); totals = new ResourceTotals() { ProviderName = resource.Controller.Type, // resource.ProviderName, ResourceName = resource.ResourceName, ResourceDescription = resource.ResourceDescription, Location = resource.Location, NodesTotal = resource.Nodes.Length, SupportedArchitectures = resource.SupportedArchitectures, }; } catch (Exception e) { Log.Warn("Can't make ResourceTotals: " + e.ToString()); totals = null; } return totals; }
public void Run(TaskSchedule schedule, IEnumerable<Resource> resources) { lock (_taskLock) { try { var execStarted = DateTime.Now; CurrentSchedule = schedule; Params[CONST.Params.Method] = Method; var resource = resources.First(r => r.ResourceName == schedule.ResourceName); string incarnatedFtpFolder = GetFtpFolder(schedule.Nodes.First(), resource, CopyPhase.In); if (PackageBaseProxy.GetSupportedPackageNames() .Any(name => String.Equals(name, Package, StringComparison.InvariantCultureIgnoreCase)) ) { //ProcessInputs(); Time.AddToOverheads(TaskTimeOverheads.InputFilesCopy, () => { Log.Debug("Uploading incarnated inputs"); foreach (var file in Incarnation.FilesToCopy) { Log.Debug(file.FileName + ": started"); IOProxy.Ftp.MakePath(incarnatedFtpFolder + Path.GetDirectoryName(file.FileName).Replace("\\", "/")); Log.Debug(file.FileName + ": path been made"); IOProxy.Storage.Download(file.StorageId, incarnatedFtpFolder + file.FileName); Log.Debug(file.FileName + ": downloaded"); } Log.Debug("Uploading incarnated inputs done"); }); } Incarnation.PackageName = Package; Incarnation.UserCert = UserCert; if (String.IsNullOrWhiteSpace(Incarnation.CommandLine)) throw new Exception("Impossible to run task with empty command line"); if (!Incarnation.CommandLine.Contains("{0}") && Incarnation.CommandLine.StartsWith(Package, StringComparison.InvariantCultureIgnoreCase)) { Incarnation.CommandLine = "{0}" + Incarnation.CommandLine.Substring(Package.Length); } Log.Stats("T_adapters", this.WfId, this.TaskId, DateTime.Now - execStarted); Time.AddToOverheads(TaskTimeOverheads.Provider, () => { //var provider = Broker.ProviderByName(resource.ProviderName); var controller = Discovery.GetControllerFarm(resource); try { //Incarnation.ProvidedTaskId = provider.Run(this.TaskId, this.Incarnation, resource, schedule.Nodes); var runContext = new ServiceProxies.ControllerFarmService.TaskRunContext() { TaskId = this.TaskId, //Incarnation = this.Incarnation, UserCert = this.UserCert, PackageName = this.Incarnation.PackageName, CommandLine = this.Incarnation.CommandLine, InputFiles = this.Incarnation.FilesToCopy.Select(f => new ServiceProxies.ControllerFarmService.FileContext() { FileName = f.FileName, StorageId = f.StorageId, }).ToList(),//.ToArray(), ExpectedOutputFileNames = this.Incarnation.ExpectedOutputFileNames.ToList(), NodesConfig = schedule.Nodes.Select(n => new ServiceProxies.ControllerFarmService.NodeRunConfig() { ResourceName = n.ResourceName, NodeName = n.NodeName, Cores = n.Cores }).ToList(),//ToArray() }; Log.Debug("Running task on controller: " + TaskId.ToString()); controller.Run(runContext); Log.Debug("Run done: " + TaskId.ToString()); controller.Close(); } catch (Exception e) { controller.Abort(); Log.Error("Exception on Task.Run for task " + this.TaskId + ": " + e.ToString()); throw; } }); State = TaskState.Started; Time.Started(TaskTimeMetric.Calculation); Log.Stats("T_clust_start", this.WfId, this.TaskId, DateTime.Now); _lastEvent = Eventing.EventType.TaskStarted; } catch (Exception e) { Log.Error(String.Format("Error on executing task {0}: {1}\n{2}", TaskId, e.Message, e.StackTrace )); Fail(reason: e.Message); } } }