public TaskSchedule(TaskSchedule other)
: this()
{
if (other != null)
{
TaskId = other.TaskId;
Action = other.Action;
var newNodes = new List<NodeConfig>();
foreach (var node in other.Nodes)
{
var newNode = node;
newNodes.Add(newNode);
}
Nodes = newNodes;
FailReason = other.FailReason;
if (other.Estimation != null)
Estimation = new Estimation(other.Estimation);
//EstimatedStartTime = other.EstimatedStartTime;
//EstimatedFinishTime = other.EstimatedFinishTime;
if (other.ModifiedParams != null)
ModifiedParams = new Dictionary<string, string>(other.ModifiedParams);
}
}
public static IEnumerable<NodeTotals> FromSchedule(TaskSchedule schedule)
{
if (schedule == null || String.IsNullOrEmpty(schedule.ResourceName) ||
schedule.Nodes == null || !schedule.Nodes.Any())
return null;
IEnumerable<NodeTotals> totals = null;
try
{
var resource = ResourceBase.GetResourceByName(schedule.ResourceName);
var res = schedule.Nodes.Select(nodeConf => new NodeTotals()
{
NodeName = nodeConf.NodeName,
NodeAddress = resource.Nodes.First(n => n.NodeName == nodeConf.NodeName).NodeAddress,
CoresUsed = (int)nodeConf.Cores,
SupportedArchitectures = resource.Nodes.First(n => n.NodeName == nodeConf.NodeName).SupportedArchitectures,
});
totals = res.ToArray();
}
catch (Exception e)
{
Log.Warn("Can't make NodeTotals: " + e.ToString());
totals = null;
}
return totals;
}
public Task(Task otherTask)
: base(otherTask)
{
if (otherTask.OutputParams != null)
OutputParams = new Dictionary<string, string>(otherTask.OutputParams);
if (otherTask.PackageEngineState != null)
this.PackageEngineState = (PackageEngineState) otherTask.PackageEngineState.Clone();
Time = new TaskTimeMeasurement(otherTask.Time);
Incarnation = new IncarnationParams(otherTask.Incarnation);
if (otherTask.Estimations != null)
Estimations = new Dictionary<string, double>(otherTask.Estimations);
CurrentSchedule = null;
if (otherTask.CurrentSchedule != null)
{
CurrentSchedule = new TaskSchedule(otherTask.CurrentSchedule);
// immutable:
this.AssignedResource = otherTask.AssignedResource;
this.AssignedNodes = otherTask.AssignedNodes;
}
State = otherTask.State;
_inputsProcessed = otherTask._inputsProcessed;
_inputsProcessingError = otherTask._inputsProcessingError;
_failReason = otherTask._failReason;
if (otherTask._lastEvent != null && otherTask._lastEvent.HasValue)
_lastEvent = otherTask._lastEvent.Value;
}
private static IEnumerable<TaskSchedule> AllocateTasks(
IEnumerable<Task> tasks, IEnumerable<Resource> resources,
// IDictionary<ulong, IEnumerable<string>> permissionsForTask
IDictionary<ulong, Dictionary<NodeConfig, Estimation>> estimationsForTask)
{
var now = DateTime.Now;
double maxCoresCount = resources.Max(r => r.Nodes.Max(n => n.CoresCount));
var availableNodes = resources
.SelectMany(r => r.Nodes.Select(n => new
{
ResourceName = n.ResourceName,
NodeName = n.NodeName,
CoresLeft = n.CoresAvailable,
TasksLeft = n.SubmissionsAvailable,
Packs = n.Packages.Select(p => p.Name.ToLowerInvariant()).ToArray()
}))
.Where(n => n.CoresLeft > 0 && n.TasksLeft > 0)
//.Where(n => n.CoresAvailable > 0)
//.OrderBy(n => n.Packages.Count + (n.CoresAvailable / maxCoresCount))
.OrderBy(n => n.Packs.Length + (n.CoresLeft / maxCoresCount))
.ToList();
// todo: log availableNodes
var scheduledTasks = new List<TaskSchedule>();
foreach (var task in tasks)
{
if (task.State == TaskState.ReadyToExecute || task.State == TaskState.Scheduled)
{
var taskSchedule = new TaskSchedule
{
TaskId = task.TaskId,
Action = ScheduledAction.None,
Nodes = new NodeConfig[0],
Estimation = new Estimation()
//EstimatedStartTime = now,
//EstimatedFinishTime = now,
};
var nodeForTask = availableNodes.FirstOrDefault(n =>
n.CoresLeft > 0 && n.TasksLeft > 0 &&
n.Packs.Contains(task.Package.ToLowerInvariant()) &&
//n.HasPackage(task.Package) &&
estimationsForTask[task.TaskId].Any(pair =>
pair.Key.ResourceName == n.ResourceName && pair.Key.NodeName == n.NodeName &&
pair.Key.Cores <= n.CoresLeft)
// permissionsForTask[task.TaskId].Contains(n.ResourceName + "." + n.NodeName, StringComparer.InvariantCultureIgnoreCase)
);
if (nodeForTask != null)
{
taskSchedule.Action = ScheduledAction.Run;
var estimation = estimationsForTask[task.TaskId]
.First(pair =>
pair.Key.ResourceName == nodeForTask.ResourceName && pair.Key.NodeName == nodeForTask.NodeName &&
pair.Key.Cores <= nodeForTask.CoresLeft)
;//.Value;
/*
taskSchedule.EstimatedStartTime = now;
/*
taskSchedule.EstimatedFinishTime =
now +
estimationsForTask[task.TaskId]
.Single(pair => pair.Key.ResourceName == nodeForTask.ResourceName && pair.Key.NodeName == nodeForTask.NodeName)
.Value;
*/
taskSchedule.Estimation = estimation.Value;
taskSchedule.Estimation.CalcStart = now;
/**/
taskSchedule.Nodes = new[]
{
new NodeConfig
{
Cores = estimation.Key.Cores, // was "1"
NodeName = nodeForTask.NodeName,
ResourceName = nodeForTask.ResourceName,
}
};
scheduledTasks.Add(taskSchedule);
availableNodes.RemoveAll(n =>
n.ResourceName == nodeForTask.ResourceName &&
n.NodeName == nodeForTask.NodeName
);
if (nodeForTask.CoresLeft > taskSchedule.Nodes.Single().Cores &&
nodeForTask.TasksLeft > 1)
{
availableNodes.Add(new
{
ResourceName = nodeForTask.ResourceName,
NodeName = nodeForTask.NodeName,
CoresLeft = (int)(nodeForTask.CoresLeft - taskSchedule.Nodes.Single().Cores),
TasksLeft = (int)(nodeForTask.TasksLeft - estimation.Key.Cores), // was "- 1"
Packs = nodeForTask.Packs
});
}
// todo: log schedule for task
}
}
}
return scheduledTasks;
/*
var limits = new[]
{
new { pack = "", otherSoft = "", maxCores = 1, maxNodes = 1},
new { pack = "GAMESS", otherSoft = "", maxCores = 1, maxNodes = 1},
new { pack = "ORCA", otherSoft = "windows", maxCores = 1, maxNodes = 1},
new { pack = "ORCA", otherSoft = "", maxCores = 8, maxNodes = 1},
new { pack = "SEMP", otherSoft = "", maxCores = 1, maxNodes = 1},
new { pack = "Plasmon", otherSoft = "", maxCores = 1, maxNodes = 1},
new { pack = "QDLaser", otherSoft = "", maxCores = 1, maxNodes = 1},
new { pack = "JAggregate", otherSoft = "", maxCores = 1, maxNodes = 1},
//new { pack = "Plasmon", otherSoft = "", maxCores = 1, maxNodes = 1},
//new { pack = "QDLaser", otherSoft = "", maxCores = 8, maxNodes = 1},
//new { pack = "JAggregate", otherSoft = "", maxCores = 8, maxNodes = 1},
//new { pack = "NanoFlow", otherSoft = "", maxCores = 16*3, maxNodes = 3},
new { pack = "NanoFlow", otherSoft = "", maxCores = 8, maxNodes = 1},
new { pack = "MD_KMC", otherSoft = "", maxCores = 8, maxNodes = 1},
new { pack = "MD_KMC", otherSoft = "windows", maxCores = 0, maxNodes = 0},
new { pack = "NTDMFT", otherSoft = "", maxCores = 8, maxNodes = 1},
new { pack = "Dalton", otherSoft = "", maxCores = 1, maxNodes = 1},
new { pack = "NAEN", otherSoft = "", maxCores = 1, maxNodes = 1},
new { pack = "Upconversion", otherSoft = "", maxCores = 8, maxNodes = 1},
new { pack = "TestP", otherSoft = "", maxCores = 1, maxNodes = 1},
new { pack = "Belman", otherSoft = "", maxCores = 8, maxNodes = 1},
new { pack = "SWAN", otherSoft = "", maxCores = 1, maxNodes = 1},
};
var scheduledTasks = new List<TaskSchedule>();
foreach (var task in tasks)
{
var packLimits = limits.Where(l => String.IsNullOrWhiteSpace(l.pack));
if (limits.Any(limit => limit.pack.ToLower() == task.Package.ToLower()))
packLimits = limits.Where(limit => limit.pack.ToLower() == task.Package.ToLower());
foreach (var resource in resources)
{
foreach (var node in resource.Nodes.Where(n => n.HasPackage(task.Package)))
{
var properPackLimit = packLimits.Max(
lim => lim.otherSoft
.Split(new char[] { ' ', '\t', '\r', '\n', ',', ';' }, StringSplitOptions.RemoveEmptyEntries)
.Intersect(node.OtherSoftware)
.Count()
);
// Choose number of cores for each node:
var coresOnNode = new List<int>();
int coresToDo = properPackLimit.maxCores;
int nodesToDo = properPackLimit.maxNodes;
foreach (var node in resource.Nodes)
{
if (coresToDo > 0 && nodesToDo > 0)
{
int coresOnCurrentNode = Math.Min(node.CoresAvailable, coresToDo);
coresOnNode.Add(coresOnCurrentNode);
coresToDo -= coresOnCurrentNode;
nodesToDo -= (coresOnCurrentNode == 0) ? 0 : 1;
}
else
coresOnNode.Add(0);
}
int coresFound = coresOnNode.Sum();
if (coresFound == 0) // haven't found anything
return null;
// Estimate (clusters with more nodes are preferable, so subtract it's nodes.Count from time estimation):
TimeSpan time = new TimeSpan((long) ((Math.Round(18000.0 / coresFound) - resource.Nodes.Count + 60)*SEC_IN_TICKS));
Estimation estimation = new Estimation(task.TaskId, resource.ProviderName, resource.ResourceName, coresOnNode.ToArray()) { ExecutionTime = time };
}
}
}
return scheduledTasks;
*/
}
public static ResourceTotals FromSchedule(TaskSchedule schedule)
{
if (schedule == null || String.IsNullOrEmpty(schedule.ResourceName))
return null;
ResourceTotals totals = null;
try
{
var resource = ResourceBase.GetResourceByName(schedule.ResourceName);
totals = new ResourceTotals()
{
ProviderName = resource.Controller.Type, // resource.ProviderName,
ResourceName = resource.ResourceName,
ResourceDescription = resource.ResourceDescription,
Location = resource.Location,
NodesTotal = resource.Nodes.Length,
SupportedArchitectures = resource.SupportedArchitectures,
};
}
catch (Exception e)
{
Log.Warn("Can't make ResourceTotals: " + e.ToString());
totals = null;
}
return totals;
}
public void Run(TaskSchedule schedule, IEnumerable<Resource> resources)
{
lock (_taskLock)
{
try
{
var execStarted = DateTime.Now;
CurrentSchedule = schedule;
Params[CONST.Params.Method] = Method;
var resource = resources.First(r => r.ResourceName == schedule.ResourceName);
string incarnatedFtpFolder = GetFtpFolder(schedule.Nodes.First(), resource, CopyPhase.In);
if (PackageBaseProxy.GetSupportedPackageNames()
.Any(name => String.Equals(name, Package, StringComparison.InvariantCultureIgnoreCase))
)
{
//ProcessInputs();
Time.AddToOverheads(TaskTimeOverheads.InputFilesCopy, () =>
{
Log.Debug("Uploading incarnated inputs");
foreach (var file in Incarnation.FilesToCopy)
{
Log.Debug(file.FileName + ": started");
IOProxy.Ftp.MakePath(incarnatedFtpFolder + Path.GetDirectoryName(file.FileName).Replace("\\", "/"));
Log.Debug(file.FileName + ": path been made");
IOProxy.Storage.Download(file.StorageId, incarnatedFtpFolder + file.FileName);
Log.Debug(file.FileName + ": downloaded");
}
Log.Debug("Uploading incarnated inputs done");
});
}
Incarnation.PackageName = Package;
Incarnation.UserCert = UserCert;
if (String.IsNullOrWhiteSpace(Incarnation.CommandLine))
throw new Exception("Impossible to run task with empty command line");
if (!Incarnation.CommandLine.Contains("{0}") &&
Incarnation.CommandLine.StartsWith(Package, StringComparison.InvariantCultureIgnoreCase))
{
Incarnation.CommandLine = "{0}" + Incarnation.CommandLine.Substring(Package.Length);
}
Log.Stats("T_adapters", this.WfId, this.TaskId, DateTime.Now - execStarted);
Time.AddToOverheads(TaskTimeOverheads.Provider, () =>
{
//var provider = Broker.ProviderByName(resource.ProviderName);
var controller = Discovery.GetControllerFarm(resource);
try
{
//Incarnation.ProvidedTaskId = provider.Run(this.TaskId, this.Incarnation, resource, schedule.Nodes);
var runContext = new ServiceProxies.ControllerFarmService.TaskRunContext()
{
TaskId = this.TaskId,
//Incarnation = this.Incarnation,
UserCert = this.UserCert,
PackageName = this.Incarnation.PackageName,
CommandLine = this.Incarnation.CommandLine,
InputFiles = this.Incarnation.FilesToCopy.Select(f => new ServiceProxies.ControllerFarmService.FileContext()
{
FileName = f.FileName,
StorageId = f.StorageId,
}).ToList(),//.ToArray(),
ExpectedOutputFileNames = this.Incarnation.ExpectedOutputFileNames.ToList(),
NodesConfig = schedule.Nodes.Select(n => new ServiceProxies.ControllerFarmService.NodeRunConfig()
{
ResourceName = n.ResourceName,
NodeName = n.NodeName,
Cores = n.Cores
}).ToList(),//ToArray()
};
Log.Debug("Running task on controller: " + TaskId.ToString());
controller.Run(runContext);
Log.Debug("Run done: " + TaskId.ToString());
controller.Close();
}
catch (Exception e)
{
controller.Abort();
Log.Error("Exception on Task.Run for task " + this.TaskId + ": " + e.ToString());
throw;
}
});
State = TaskState.Started;
Time.Started(TaskTimeMetric.Calculation);
Log.Stats("T_clust_start", this.WfId, this.TaskId, DateTime.Now);
_lastEvent = Eventing.EventType.TaskStarted;
}
catch (Exception e)
{
Log.Error(String.Format("Error on executing task {0}: {1}\n{2}",
TaskId, e.Message, e.StackTrace
));
Fail(reason: e.Message);
}
}
}
