public IEnumerable<NodeStateResponse> GetNodesState(Resource resource)
{
var partitionNames = resource.Nodes.Select(n => GetPartition(n)).Distinct();
var badPartitions = new List<string>();
foreach (string partitionName in partitionNames)
{
string result = SshExec(resource.Nodes.First(), SshCommands.GetPartitionState, partitionName);
string result_LOWER = result.ToLowerInvariant();
if (!result_LOWER.Contains("state=up") ||
result_LOWER.Contains("totalnodes=0 ") ||
result_LOWER.Contains("totalcpus=0 "))
{
badPartitions.Add(partitionName);
}
}
var nodeStates = resource.Nodes.Select(n =>
badPartitions.Contains(GetPartition(n)) ?
new NodeStateResponse(n.NodeName) { State = NodeState.Down } :
new NodeStateResponse(n.NodeName) { State = NodeState.Available }
);
return nodeStates;
}
//public static string GetNewProvidedTaskId()
//{
// var service = EntryPointProxy.GetClustersService();
// ClustersService.Code errCode;
// ServiceProxies.ClustersService.TaskInfo taskInfo = null;
// lock (_clustersServiceLock)
// {
// taskInfo = service.CreateTask(out errCode);
// }
// if (errCode == ServiceProxies.ClustersService.Code.OperationSuccess)
// return taskInfo.TaskID;
// else
// throw new ClusterException(errCode);
//}
/// <summary>
/// Get resource-specific task state and convert it to one of task's possible states
/// </summary>
/// <param name="providedTaskId">Task’s id specific to this resource provider</param>
/// <returns>Tuple (task state, "fail reason or some comment to task's state")</returns>
public override Tuple<TaskState, string> GetTaskState(ulong taskId, string providedTaskId, Resource resource, IEnumerable<NodeConfig> nodesConfig)
{
var service = EntryPointProxy.GetClustersService();
ClustersService.Code errCode;
ClustersService.TaskInfo taskInfo;
lock (_clustersServiceLock)
{
taskInfo = service.GetTaskState(providedTaskId, out errCode);
}
if (errCode != ServiceProxies.ClustersService.Code.OperationSuccess)
throw new ClusterException(errCode);
if (taskInfo.State == ClustersService.TaskState.Complete)
return Tuple.Create(TaskState.Completed, "");
if (taskInfo.State == ClustersService.TaskState.Fail)
return Tuple.Create(TaskState.Failed, "Failed on cluster");
if (taskInfo.State == ClustersService.TaskState.Execute)
return Tuple.Create(TaskState.Started, "");
if (taskInfo.State == ClustersService.TaskState.Cancel)
return Tuple.Create(TaskState.Aborted, "");
return Tuple.Create(TaskState.Defined, taskInfo.State.ToString());
}
protected override bool Matches(TaskDescription task, Resource resource)
{
/*
if (nodes.First().ClusterName.ToLower() == "cluster_niinkt_1")
return true;
/**/
return false;
}
public virtual Estimation EstimateIfMatches(TaskDescription task, Resource resource)
{
// todo : EstimateIfMatches => try/catch & nodes != null & nodes.Any() & nodes.First().HavePackage(task.Package)
if (Matches(task, resource))
return Estimate(task, resource);
return null;
}
protected override Estimation Estimate(TaskDescription task, Resource resource)
{
// get all available nodes with all their cores
int n = resource.Nodes.Sum(node => node.CoresAvailable);
int[] cores = resource.Nodes.Select(node => node.CoresAvailable).ToArray();
long sec = (long) 1e7;
TimeSpan time = new TimeSpan((long) (240.0 * sec / (1.0 * n)));
return new Estimation(task.TaskId, resource.ProviderName, resource.ResourceName, cores) { ExecutionTime = time };
}
public override void Abort(string providedTaskId, Resource resource, IEnumerable<NodeConfig> nodesConfig)
{
lock (_hadoopLock)
{
var node = GetDefaultNodeSettings(resource, nodesConfig);
string host = node.Services.ExecutionUrl;
SshRun(host, HadoopCommands.Abort);
_nodeUsed = false;
}
}
//private void ApplyNodesState
private ResourceCache(Resource resource, IStatelessResourceController controller, INodeGlobalCacheCollector collector = null )
{
Resource = resource;
Controller = controller;
gcCollector = collector;
NodeStateInfo = resource.Nodes.Select(node =>
new NodeStateInfo(node.ResourceName, node.NodeName, node.CoresCount, node.TasksSubmissionLimit)
).ToArray();
_isUpdating = false;
_lastUpdateTime = DateTime.Now - UPDATE_INTERVAL - TimeSpan.FromMilliseconds(50);
}
protected ResourceNode GetNode(Resource resource, IEnumerable<NodeRunConfig> nodesConfig)
{
if (nodesConfig.Count() > 1)
{
Log.Warn(String.Format(
"More than one node scheduled, but resource provider doesn’t support different settings on different nodes. Using settings from node '{0}'.",
nodesConfig.First().NodeName
));
}
var node = resource.Nodes.First(n => n.NodeName == nodesConfig.First().NodeName);
return node;
}
protected override Estimation Estimate(TaskDescription task, Resource resource)
{
// get 1 core on 1 node, less time estimated
int n = 0;
int[] cores = Enumerable.Range(0, resource.Nodes.Count).ToArray();
cores[0] = 1;
long sec = (long) 1e7;
TimeSpan time = new TimeSpan((long) (0.5 * n * 240.0 * sec / (1.0)));
return new Estimation(task.TaskId, resource.ProviderName, resource.ResourceName, cores) { ExecutionTime = time };
}
public override void Abort(string providedTaskId, Resource resource, IEnumerable<NodeConfig> nodesConfig)
{
lock (_pcLock)
{
string[] providedWords = providedTaskId.Split(new char[] { '\n' });
if (providedWords.Length > 2)
Log.Warn(String.Format("Too many sections in provided task id for win PC: {0}", providedTaskId));
string pid = providedWords[0];
string nodeName = providedWords[1];
_nodeUsed[nodeName] = false; // todo: do a real abortion
}
}
public IEnumerable<NodeStateResponse> GetNodesState(Resource resource)
{
var nodesResp = new List<NodeStateResponse>();
var respLock = new object();
PFX.Parallel.ForEach(resource.Nodes, node =>
{
var rexService = GetREx(node.Services.ExecutionUrl);
rexService.InnerChannel.OperationTimeout = TimeSpan.FromSeconds(5);
try
{
bool isRunning = rexService.IsProcessRunning(1024);
rexService.Close();
lock (respLock)
{
nodesResp.Add(new NodeStateResponse(node.NodeName)
{
State = NodeState.Available
});
}
}
catch (Exception e)
{
rexService.Abort();
logger.Debug("Node {0}.{1} is down: {2}", node.ResourceName, node.NodeName, e);
lock (respLock)
{
nodesResp.Add(new NodeStateResponse(node.NodeName)
{
State = NodeState.Down
});
}
}
});
/*
var nodes = resource.Nodes.Select(n => new NodeStateResponse(n.NodeName)
{
State = NodeState.Available
});
*/
return nodesResp;
}
public new IEnumerable<NodeStateResponse> GetNodesState(Resource resource)
{
var nodeStates = new List<NodeStateResponse>();
foreach(var node in resource.Nodes)
{
string responseString = SshExec(node, SshUnixCommands.GetNodeState).ToLowerInvariant();
Log.Debug("Load response: " + responseString);
/*
string LOAD_STRING_START = "load average: ";
string LOAD_STRING_END = ",";
int loadStartPos = responseString.IndexOf(LOAD_STRING_START) + LOAD_STRING_START.Length;
int loadEndPos = responseString.IndexOf(LOAD_STRING_END, loadStartPos);
string parseStr = responseString.Substring(loadStartPos, loadEndPos - loadStartPos);
Log.Debug("Parsing val = '" + parseStr + "'");
//double val = double.Parse(responseString.Substring(0, responseString.Length - 1), CultureInfo.InvariantCulture);
double val = double.Parse(parseStr, CultureInfo.InvariantCulture);
Log.Debug("Load value = " + val.ToString());
*/
var rx = new Regex(@": (\d+[,.]\d*), (\d+[,.]\d*), (\d+[,.]\d*)\s*$");
string loadStr = rx.Match(responseString).Groups[1].Value.Replace(',', '.');
Log.Debug("Load string: " + loadStr);
double loadValue = double.Parse(loadStr, CultureInfo.InvariantCulture);
Log.Debug("Load value: " + loadValue.ToString());
var state = (loadValue < BUSY_RANGE)
? new NodeStateResponse(node.NodeName) {State = NodeState.Available}
: new NodeStateResponse(node.NodeName) {State = NodeState.Busy};
//todo nbutakov change
state.Consumption = new List<NodeInfo>();
state.Consumption.Add(GetCurrentResourceInfo(node));
nodeStates.Add(state);
}
return nodeStates;
}
public override Tuple<TaskState, string> GetTaskState(ulong taskId, string providedTaskId, Resource resource, IEnumerable<NodeConfig> nodesConfig)
{
lock (_hadoopLock)
{
var node = GetDefaultNodeSettings(resource, nodesConfig);
string host = node.Services.ExecutionUrl;
string state = SshRun(host, String.Format(HadoopCommands.GetTaskState, providedTaskId));
if (state != "")
{
return new Tuple<TaskState, string>(TaskState.Started, "");
}
else
{
_nodeUsed = false;
return new Tuple<TaskState, string>(TaskState.Completed, "");
}
}
}
public static IStatelessResourceController Build(Resource resource)
{
try
{
var assembly = Assembly.GetAssembly(typeof(IStatelessResourceController));
var controllerTypes = assembly.GetExportedTypes().Where(type => type.IsClass && !type.IsAbstract && type.GetInterfaces().Contains(typeof(IStatelessResourceController)));
var controllerType = controllerTypes.Single(t => t.Name.ToLowerInvariant() == resource.Controller.Type.ToLowerInvariant());
var controller = (IStatelessResourceController)controllerType.GetConstructor(Type.EmptyTypes).Invoke(null);
return controller;
}
catch (Exception e)
{
Log.Error(String.Format("Exception while building controller: {0}\n{1}",
e.Message, e.StackTrace
));
throw new Exception("Unknown controller " + resource.Controller.Type, e);
}
}
public IEnumerable<NodeStateResponse> GetNodesState(Resource resource)
{
string responseString = SshExec(resource.Nodes.First(), GetNodeStateCommand()).ToLowerInvariant();
var responseXml = XElement.Parse(responseString);
var freeNodeNames = responseXml.Descendants("node")
.Where(node => (node.Element("state").Value.Equals("free")))
.Select(node => (node.Element("name").Value));
/*var freeNodeNames = responseXml.Descendants("node")
.Where(node => node.Element("status").Value.Contains("state=free"))
.Select(node => node.Element("name").Value);*/
var nodeStates = resource.Nodes.Select(node =>
freeNodeNames.Contains(node.NodeName) ?
new NodeStateResponse(node.NodeName) { State = NodeState.Available } :
new NodeStateResponse(node.NodeName) { State = NodeState.Busy }
);
return nodeStates;
}
public override Tuple<TaskState, string> GetTaskState(ulong taskId, string providedTaskId, Resource resource, IEnumerable<NodeConfig> nodesConfig)
{
lock (_pcLock)
{
try
{
string[] providedWords = providedTaskId.Split(new char[] { '\n' });
if (providedWords.Length > 2)
Log.Warn(String.Format("Too many sections in provided task id for win PC: {0}", providedTaskId));
string pid = providedWords[0];
string nodeName = providedWords[1];
var node = resource.Nodes.First(n => n.NodeName == nodeName);
using (var rexService = EntryPointProxy.GetREx(node.Services.ExecutionUrl))
{
bool isRunning = rexService.IsProcessRunning(Int32.Parse(pid));
if (!isRunning)
{
_nodeUsed[node.NodeName] = false;
return Tuple.Create(TaskState.Completed, "");
}
return Tuple.Create(TaskState.Started, "");
}
}
catch (Exception e)
{
Log.Warn(String.Format(
"Exception while getting task state (provided id = {0}): {1}\n{2}",
providedTaskId, e.Message, e.StackTrace
));
return Tuple.Create(TaskState.Started, "");
}
}
}
public static Resource[] GetAllResources(bool updateThem = false)
{
var resourceLoadTimer = System.Diagnostics.Stopwatch.StartNew();
var resourceBase = Discovery.GetResourceBase();
Resource[] resources = new Resource[0];
try
{
resources = resourceBase.GetAllResources();
if (updateThem)
{
//foreach (var resource in resources)
PFX.Parallel.ForEach(resources, (resource) =>
{
//var updateStarted = DateTime.Now;
var controller = Discovery.GetControllerFarm(resource);
try
{
var nodesStateInfo = controller.GetNodesState(resource.ResourceName);
foreach (var nodeState in nodesStateInfo)
{
var node = resource.Nodes.Single(n => n.NodeName == nodeState.NodeName);
node.CoresAvailable = nodeState.CoresAvailable;
node.SubmissionsAvailable = (int) Math.Max(0, nodeState.TasksSubmissionLimit - nodeState.TasksSubmitted);
}
string[] availableNodeNames = nodesStateInfo.Where(n => n.State == ControllerFarmService.NodeState.Available).Select(n => n.NodeName).ToArray();
//resource.Nodes = resource.Nodes.Where(n => availableNodeNames.Contains(n.NodeName)).ToArray();
controller.Close();
}
catch (Exception e)
{
controller.Abort();
Log.Warn(String.Format("Exception while updating resource '{0}' state: {1}", resource.ResourceName, e));
resource.Nodes = new ResourceNode[0]; // should not count any nodes as 'will be available sometime'
/*
foreach (var node in resource.Nodes)
{
node.CoresAvailable = 0;
node.SubmissionsAvailable = 0;
}
*/
}
finally
{
//var updateFinished = DateTime.Now;
//resourceUpdateTime += updateFinished - updateStarted;
}
});
resourceLoadTimer.Stop();
Log.Info(String.Format(
"Resources loaded and updated in {0} seconds",
resourceLoadTimer.Elapsed.TotalSeconds
));
}
resourceBase.Close();
}
catch (Exception e)
{
resourceBase.Abort();
if (resourceLoadTimer.IsRunning)
resourceLoadTimer.Stop();
Log.Error("Exception on getting all resources from ResourceBase: " + e.ToString());
throw;
}
if (updateThem)
{
resources = resources.Where(r => r.Nodes.Any()).ToArray();
Log.Info("Available resources: " + String.Join(", ", resources.Select(r => r.ResourceName + "(" + String.Join(", ", r.Nodes.Select(n => n.NodeName)) + ")")));
}
return resources;
}
protected override Estimation Estimate(TaskDescription task, Resource resource)
{
return new Estimation(
task.TaskId,
resource.ProviderName,
resource.ResourceName,
new int[1] { 1 }
) {
ExecutionTime = new TimeSpan(0),
};
}
public override void Abort(string providedTaskId, Resource resource, IEnumerable<NodeConfig> nodesConfig)
{
// todo : [5] implement ClustersProxy.Abort(clusterName)
var service = EntryPointProxy.GetClustersService();
ClustersService.Code errCode;
lock (_clustersServiceLock)
{
service.CancelTask(providedTaskId, out errCode);
}
if (errCode != ClustersService.Code.OperationSuccess)
throw new ClusterException(errCode);
}
public override string Run(ulong taskId, IncarnationParams incarnation, Resource resource, IEnumerable<NodeConfig> nodesConfig)
{
lock (_hadoopLock)
{
var node = GetDefaultNodeSettings(resource, nodesConfig);
var pack = node.PackageByName(incarnation.PackageName);
string workDir = IncarnationParams.IncarnatePath(node.DataFolders.LocalFolder, taskId, CopyPhase.None);
string workScriptPath = workDir + "hadoop.sh";
string workScriptInternalPath = workDir + "hadoop_internal.sh";
string exchangeInDir = IncarnationParams.IncarnatePath(node.DataFolders.ExchangeUrlFromResource, taskId, CopyPhase.In);
string exchangeOutDir = IncarnationParams.IncarnatePath(node.DataFolders.ExchangeUrlFromResource, taskId, CopyPhase.Out);
string identity = exchangeInDir.Split('@')[0].Replace("ftp://", "");
string user = identity.Split(':')[0];
string pass = identity.Split(':')[1];
string hostAndPath = exchangeInDir.Split('@')[1];
string ftpHost = hostAndPath.Split(new char[] { '/' }, 2)[0];
string ftpInPath = hostAndPath.Split(new char[] { '/' }, 2)[1];
string ftpOutPath = exchangeOutDir.Split('@')[1].Split(new char[] { '/' }, 2)[1];
//pack.Params.
var cmd = new StringBuilder();
cmd.AppendFormat("cd " + workDir + "\n");
cmd.AppendFormat(String.Format(HadoopCommands.Run, workScriptInternalPath) + "\n");
cmd.AppendFormat("echo $!" + "\n");
string tempPath = Path.GetTempFileName();
File.WriteAllText(tempPath, cmd.ToString());
var cmd_internal = new StringBuilder();
cmd_internal.AppendFormat("ftp -n {0} << END_SCRIPT\n", ftpHost);
cmd_internal.AppendFormat("quote User {0}\n", user);
cmd_internal.AppendFormat("quote PASS {0}\n", pass);
foreach (TaskFileDescription fileDesc in incarnation.FilesToCopy)
{
cmd_internal.AppendFormat("get {0}{2} {1}{2}\n", ftpInPath, workDir, fileDesc.FileName);
}
cmd_internal.AppendFormat("quit" + "\n");
cmd_internal.AppendFormat("END_SCRIPT" + "\n");
cmd_internal.AppendFormat(String.Format(incarnation.CommandLine, IncarnationParams.IncarnatePath(pack.AppPath, taskId, CopyPhase.None).TrimEnd('/')) + "\n");
cmd_internal.AppendFormat("ftp -n {0} << END_SCRIPT\n", ftpHost);
cmd_internal.AppendFormat("quote User {0}\n", user);
cmd_internal.AppendFormat("quote PASS {0}\n", pass);
cmd_internal.AppendFormat("mkdir {0}\n", ftpOutPath);
foreach (string fileName in incarnation.ExpectedOutputFileNames)
{
cmd_internal.AppendFormat("put {0}{2} {1}{2}\n", workDir, ftpOutPath, fileName);
}
cmd_internal.AppendFormat("quit" + "\n");
cmd_internal.AppendFormat("END_SCRIPT" + "\n");
string tempPathInternal = Path.GetTempFileName();
File.WriteAllText(tempPathInternal, cmd_internal.ToString());
string host = node.Services.ExecutionUrl;
SshRun(host, "mkdir " + workDir, true); // Need check, del
SshRun(host, "rm " + workDir + "*", true); // Need check
foreach (string path in pack.CopyOnStartup) // todo : internal_script maybe?
{
string toCopy = (path.EndsWith("/")? path + "*": path); // 'folder/*' needed for contents copy
// ^^^^^ doesn't work everywhere
SshRun(host, "cp -fpR " + toCopy + "/* " + workDir);
}
/*
SshRun(host, "cp " + "/home/clavire/hadrawler/clavire.sh " + workDir); // Need check, del
SshRun(host, "cp " + "/home/clavire/hadrawler/hadoop2.conf " + workDir); // Need check, del
*/
ScpPut(host, tempPath, workScriptPath);// Need del
SshRun(host, "chmod 700 " + workScriptPath);
ScpPut(host, tempPathInternal, workScriptInternalPath);// Need del
SshRun(host, "chmod 700 " + workScriptInternalPath);
File.Delete(tempPath);
File.Delete(tempPathInternal);
string pid = SshShell(host, workScriptPath); // проверка на то запустилось ли
_nodeUsed = true;
return pid.Split('\n')[2].TrimEnd('\r');
}
}
protected override Estimation Estimate(TaskDescription task, Resource resource)
{
if (resource.ProviderName == CONST.Providers.WinPc)
{
var node = resource.Nodes.First();
if (node.HasPackage(task.Package) && node.CoresAvailable > 0)
{
string nodeName = node.NodeName;
return new Estimation(
task.TaskId,
resource.ProviderName,
resource.ResourceName,
resource.Nodes.Select(n => (n.NodeName == nodeName)? 1 : 0).ToArray()
)
{
//ExecutionTime = new TimeSpan(Math.Max(0, 500 - nodes.Length)*SEC_IN_TICKS),
ExecutionTime = TimeSpan.FromSeconds(2),
};
}
}
return null;
}
public double Estimate(Task task, Resource resource)
{
return((task.mean + resource.mean) / (1.0 * resource.performance) + resource.estimatedAvailabilityTime);
}
public IEnumerable<NodeStateResponse> GetNodesState(Resource resource)
{
lock (_gridLock)
{
var response = resource.Nodes.Select(n => new NodeStateResponse(n.NodeName)
{
State = NodeState.Available
});
return response.ToArray();
}
}
protected override Estimation Estimate(TaskDescription task, Resource resource)
{
if (task == null || resource == null || !resource.Nodes.Any())
return null;
var limits = new[]
{
new { pack = "", otherSoft = "", maxCores = 1, maxNodes = 1},
new { pack = "GAMESS", otherSoft = "", maxCores = 1, maxNodes = 1},
new { pack = "ORCA", otherSoft = "windows", maxCores = 1, maxNodes = 1},
new { pack = "ORCA", otherSoft = "", maxCores = 8, maxNodes = 1},
new { pack = "SEMP", otherSoft = "", maxCores = 1, maxNodes = 1},
new { pack = "Plasmon", otherSoft = "", maxCores = 1, maxNodes = 1},
new { pack = "QDLaser", otherSoft = "", maxCores = 1, maxNodes = 1},
new { pack = "JAggregate", otherSoft = "", maxCores = 1, maxNodes = 1},
//new { pack = "Plasmon", otherSoft = "", maxCores = 1, maxNodes = 1},
//new { pack = "QDLaser", otherSoft = "", maxCores = 8, maxNodes = 1},
//new { pack = "JAggregate", otherSoft = "", maxCores = 8, maxNodes = 1},
//new { pack = "NanoFlow", otherSoft = "", maxCores = 16*3, maxNodes = 3},
new { pack = "NanoFlow", otherSoft = "", maxCores = 8, maxNodes = 1},
new { pack = "MD_KMC", otherSoft = "", maxCores = 8, maxNodes = 1},
new { pack = "MD_KMC", otherSoft = "windows", maxCores = 0, maxNodes = 0},
new { pack = "NTDMFT", otherSoft = "", maxCores = 8, maxNodes = 1},
new { pack = "Dalton", otherSoft = "", maxCores = 1, maxNodes = 1},
new { pack = "NAEN", otherSoft = "", maxCores = 1, maxNodes = 1},
new { pack = "Upconversion", otherSoft = "", maxCores = 8, maxNodes = 1},
new { pack = "TestP", otherSoft = "", maxCores = 1, maxNodes = 1},
new { pack = "Belman", otherSoft = "", maxCores = 8, maxNodes = 1},
new { pack = "SWAN", otherSoft = "", maxCores = 1, maxNodes = 1},
};
var packLimits = limits.Where(l => String.IsNullOrWhiteSpace(l.pack));
if (limits.Any(limit => limit.pack.ToLower() == task.Package.ToLower()))
packLimits = limits.Where(limit => limit.pack.ToLower() == task.Package.ToLower());
// selecting proper limit (maximize software matches):
var properPackLimit = packLimits.First();
int maxSoftMatches = 0;
foreach (var packLimit in packLimits)
{
int softMathcesCount = 0;
string[] otherSoft = packLimit.otherSoft.Split(new char[] { ' ', '\t', '\r', '\n', ',', ';' }, StringSplitOptions.RemoveEmptyEntries);
foreach (string soft in otherSoft)
{
// todo : [future] packs on different nodes can be different
if (resource.Nodes.First().OtherSoftware.Contains(soft.ToLower()))
softMathcesCount++;
}
if (softMathcesCount > maxSoftMatches || (packLimit.maxCores > properPackLimit.maxCores && softMathcesCount == maxSoftMatches))
{
maxSoftMatches = softMathcesCount;
properPackLimit = packLimit;
}
}
// Choose number of cores for each node:
var coresOnNode = new List<int>();
int coresToDo = properPackLimit.maxCores;
int nodesToDo = properPackLimit.maxNodes;
foreach (var node in resource.Nodes)
{
if (coresToDo > 0 && nodesToDo > 0)
{
int coresOnCurrentNode = Math.Min(node.CoresAvailable, coresToDo);
coresOnNode.Add(coresOnCurrentNode);
coresToDo -= coresOnCurrentNode;
nodesToDo -= (coresOnCurrentNode == 0) ? 0 : 1;
}
else
coresOnNode.Add(0);
}
int coresFound = coresOnNode.Sum();
if (coresFound == 0) // haven't found anything
return null;
// Estimate (clusters with more nodes are preferable, so subtract it's nodes.Count from time estimation):
TimeSpan time = new TimeSpan((long) ((Math.Round(18000.0 / coresFound) - resource.Nodes.Count + 60)*SEC_IN_TICKS));
Estimation estimation = new Estimation(task.TaskId, resource.ProviderName, resource.ResourceName, coresOnNode.ToArray())
{ ExecutionTime = time };
return estimation;
}
public override string Run(ulong taskId, IncarnationParams incarnation, Resource resource, IEnumerable<NodeConfig> nodesConfig)
{
string providedTaskId = taskId.ToString();
var node = GetDefaultNodeSettings(resource, nodesConfig);
var pack = node.PackageByName(incarnation.PackageName);
var service = EntryPointProxy.GetClustersService();
ClustersService.Code errCode;
ClustersService.TaskInfo taskInfo;
lock (_clustersServiceLock)
{
taskInfo = service.GetTaskState(providedTaskId, out errCode);
}
if (errCode != ServiceProxies.ClustersService.Code.OperationSuccess)
throw new ClusterException(errCode);
taskInfo.ClusterName = resource.ResourceName;
taskInfo.CommandLine = String.Format(incarnation.CommandLine, pack.AppPath);
taskInfo.PackageName = incarnation.PackageName.ToUpperInvariant();
/*
if (!String.IsNullOrEmpty(incarnation.StdInFile))
taskInfo.StdinFileName = incarnation.StdInFile;
else
taskInfo.StdinFileName = "";
if (!String.IsNullOrEmpty(incarnation.StdOutFile))
taskInfo.StdoutFileName = incarnation.StdOutFile;
else
taskInfo.StdoutFileName = "";
*/
// cores on nodes: {n, 0, 0} -> {n}
taskInfo.NumberOfCores = new ClustersService.ArrayOfInt();
taskInfo.NumberOfCores.AddRange(nodesConfig.Where(conf => conf.Cores > 0).Select(conf => conf.Cores));
taskInfo.NumberOfNodes = taskInfo.NumberOfCores.Count;
var logStream = new StringWriter();
logStream.WriteLine("Задача {0} ({1}) запускается на кластере {2}",
taskInfo.TaskID, taskInfo.PackageName, taskInfo.ClusterName);
logStream.WriteLine(" Папка с файлами расчета: {0}", taskInfo.FTPPath);
logStream.WriteLine(" Строка запуска: {0}", taskInfo.CommandLine);
logStream.WriteLine(" Перенаправление вывода: {0}", taskInfo.StdoutFileName);
logStream.Write(" Количество ядер (по каждому узлу): ");
foreach (int coresCount in taskInfo.NumberOfCores)
logStream.Write("{0} ", coresCount);
Log.Info(logStream.ToString());
lock (_clustersServiceLock)
{
errCode = service.ExecuteTask(taskInfo);
}
if (errCode != ServiceProxies.ClustersService.Code.OperationSuccess)
throw new ClusterException(String.Format(
CONST.Dirty<string>("Ошибка интегратора управления кластерами при запуске задачи: {0}"), errCode.ToString()
));
return providedTaskId;
}
private Resource[] GetAllowed(Resource[] resources, string userId)
{
if (CheckDebugMode())
{
return resources;
}
var uris = GetUriesForRead(userId);
var names = uris.Select(x => x.Name);
return resources.Where(x => names.Contains(x.ResourceName)).ToArray();
}
protected override bool Matches(TaskDescription task, Resource resource)
{
return false;
}
private Tuple<TaskState, string> GetTaskStateByTaskList(string providedTaskId, Resource resource)
{
lock (_pcLock)
{
//AcceptPsToolsEula();
string[] providedWords = providedTaskId.Split( new char[] {'\n'} );
if (providedWords.Length > 2)
Log.Warn(String.Format("Too many sections in provided task id for win PC: {0}", providedTaskId));
string pid = providedWords[0];
string nodeName = providedWords[1];
var node = resource.Nodes.First(n => n.NodeName == nodeName);
// todo : state == defined?
//string taskIdString = taskId.ToString();
var taskListProcess = new Process();
taskListProcess.StartInfo.UseShellExecute = false;
taskListProcess.StartInfo.RedirectStandardOutput = true;
taskListProcess.StartInfo.RedirectStandardError = true;
/*
taskListProcess.StartInfo.FileName = CONST.Path.PsList;
taskListProcess.StartInfo.Arguments = String.Format(
@"\\{0} -u {1} -p {2} {3}",
node.NodeAddress, node.Credentials.Username, node.Credentials.Password, pid
);
*/
taskListProcess.StartInfo.FileName = "tasklist.exe";
taskListProcess.StartInfo.Arguments = String.Format(
@"/s {0} /u {1} /p {2} /fo csv /nh /fi " + "\"PID eq {3}\"",
node.NodeAddress, node.Credentials.Username, node.Credentials.Password, pid
);
taskListProcess.Start();
string table = taskListProcess.StandardOutput.ReadToEnd();
taskListProcess.WaitForExit();
Log.Debug(String.Format("Process table for machine {0}:\n{1}", node.NodeAddress, table));
if (!String.IsNullOrWhiteSpace(table))
{
int PID_COLUMN_NUM = 1;
var pids = table
.Split(new char[] { '\r', '\n' }, StringSplitOptions.RemoveEmptyEntries) // rows
.Select(row => row
.Split(new char[] { ',' }) // cols
.ElementAt(PID_COLUMN_NUM) // "pid"
.Trim(new char[] { ' ', '\t', '\"' }) // pid
);
if (!pids.Contains(pid))
{
_nodeUsed[node.NodeName] = false;
return Tuple.Create(TaskState.Completed, "");
}
}
return Tuple.Create(TaskState.Started, "");
/*
string state = taskListProcess.StandardOutput.ReadToEnd() + " " + taskListProcess.StandardError.ReadToEnd();
taskListProcess.WaitForExit();
if (state.Contains(PS_PROCESS_NOT_FOUND_MSG))
{
_nodeUsed[node.NodeName] = false;
return Tuple.Create(TaskState.Completed, "");
}
return Tuple.Create(TaskState.Started, state);
*/
}
}
public override string Run(ulong taskId, IncarnationParams incarnation, Resource resource, IEnumerable<NodeConfig> nodesConfig)
{
lock (_pcLock)
{
//AcceptPsToolsEula();
int coresToUse = nodesConfig.Sum(conf => conf.Cores);
var node = GetDefaultNodeSettings(resource, nodesConfig);
if (_nodeUsed[node.NodeName])
throw new Exception(String.Format("Could not run task {0} on node {1}: node used by another task", taskId, node.NodeName));
string ftpFolder = IncarnationParams.IncarnatePath(node.DataFolders.ExchangeUrlFromSystem, taskId, CopyPhase.In);
string jobFtpFolder = IncarnationParams.IncarnatePath(node.DataFolders.ExchangeUrlFromSystem, taskId, CopyPhase.None);
string sharedInputFolder = IncarnationParams.IncarnatePath(node.DataFolders.ExchangeUrlFromResource, taskId, CopyPhase.In);
string sharedOutputFolder = IncarnationParams.IncarnatePath(node.DataFolders.ExchangeUrlFromResource, taskId, CopyPhase.Out);
string tmpFolder = IncarnationParams.IncarnatePath(node.DataFolders.LocalFolder, taskId, CopyPhase.None);
IOProxy.Ftp.MakePath(ftpFolder);
IOProxy.Ftp.MakePath(jobFtpFolder);
string jobFileName = "job_" + taskId + ".cmd";
Log.Info(String.Format(
"Trying to exec task {0} on win PC {1}",
taskId, node.NodeName
));
var pack = node.Packages.First(p => String.Equals(p.Name, incarnation.PackageName, StringComparison.InvariantCultureIgnoreCase));
string batchContent = "";
batchContent += "mkdir " + tmpFolder.TrimEnd(new char[] { '/', '\\' }) + Environment.NewLine;
if (Path.IsPathRooted(tmpFolder)) // change drive if needed
batchContent += Path.GetPathRoot(tmpFolder).TrimEnd(new char[] { '/', '\\' }) + Environment.NewLine;
batchContent += String.Format(
@"cd {0}" + Environment.NewLine,
tmpFolder.TrimEnd(new char[] { '/', '\\' })
);
batchContent += "echo %time% > clavire_script_started" + Environment.NewLine;
foreach (string copyPath in pack.CopyOnStartup)
batchContent += String.Format(
@"xcopy {0} {1}\ /z /s /e /c /i /h /r /y" + Environment.NewLine,
copyPath.TrimEnd(new char[] { '/', '\\' }),
tmpFolder.TrimEnd(new char[] { '/', '\\' })
);
batchContent += String.Format(
//@"ping localhost -w 1000 -n 50" + Environment.NewLine +
@"xcopy {0} {1}\ /z /s /e /c /i /h /r /y" + Environment.NewLine,
sharedInputFolder.TrimEnd(new char[] {'/', '\\'}),
tmpFolder.TrimEnd(new char[] { '/', '\\' })
);
// todo : env vars on WinPc provider
string commandLine = incarnation.CommandLine;
//var pack = node.Packages.First(p => commandLine.StartsWith(p.Name, StringComparison.InvariantCultureIgnoreCase));
//commandLine = pack.Params["appPath"] + commandLine.Substring(pack.Name.Length);
commandLine = String.Format(incarnation.CommandLine, pack.AppPath);
//commandLine = String.Format(incarnation.CommandLine, pack.Params["appPath"]);
batchContent += "echo %time% > clavire_task_started" + Environment.NewLine;
batchContent += //"start \"" + jobFileName + " " + incarnation.PackageNameInConfig + "\" /wait /b" +
"cmd.exe /c " + commandLine + Environment.NewLine;
batchContent += "echo %time% > clavire_task_finished" + Environment.NewLine;
foreach (string delPath in pack.Cleanup)
{
batchContent += String.Format(
@"rmdir /s /q {0}" + Environment.NewLine +
@"del /f /s /q {0}" + Environment.NewLine,
tmpFolder + delPath
);
}
batchContent += String.Format(
@"xcopy {1} {0}\ /z /s /e /c /i /h /r /y" + Environment.NewLine,
sharedOutputFolder.TrimEnd(new char[] { '/', '\\' }),
tmpFolder.TrimEnd(new char[] { '/', '\\' })
);
batchContent += String.Format(
@"ping localhost -n 3" + Environment.NewLine +
@"echo %time% > clavire_script_finished" + Environment.NewLine +
@"xcopy clavire_script_finished {1}\ /z /s /e /c /i /h /r /y" + Environment.NewLine +
@"cd {0}" + Environment.NewLine +
@"cd .." + Environment.NewLine +
//@"rmdir /s /q {0}" + Environment.NewLine +
"",
tmpFolder.TrimEnd(new char[] { '/', '\\' }),
sharedOutputFolder.TrimEnd(new char[] { '/', '\\' })
);
IOProxy.Ftp.UploadFileContent(batchContent, jobFtpFolder, jobFileName);
//string cmdArgs = "/c " + CONST.Path.PsExec.Replace("PsExec.exe", "p.cmd");
//string cmdArgs = "\\\\192.168.4.1 -u nano -p Yt1NyDpQNm -d cmd.exe /c \"\\\\192.168.4.1\\ftp_exchange\\Tasks\\10043\\job_10043.cmd\"";
//Log.Debug(cmdArgs);
//Process.Start(CONST.Path.PsExec, cmdArgs);
//**/
//var psexecProcess = new Process();
//psexecProcess.StartInfo.UseShellExecute = false;
////psexecProcess.StartInfo.RedirectStandardOutput = true;
////psexecProcess.StartInfo.RedirectStandardError = true;
//psexecProcess.StartInfo.FileName = CONST.Path.PsExec;
//psexecProcess.StartInfo.Arguments = String.Format(
// "\\\\{0} -d -u {1} -p {2} cmd.exe /c {4}", // -d -w \"{3}\" ^> C:\\Temp\\out
// //"-u nano -p Yt1NyDpQNm cmd.exe /c " + CONST.Path.PsExec.Replace("PsExec.exe", "p.cmd"),
// resParams.name, resParams.user, resParams.pass,
// resParams.tempFolderOnMachine.Replace(@"\", @"\\"),
// sharedJobFilePath
//);
//*
//psexecProcess.StartInfo.UserName = "******";
//psexecProcess.StartInfo.Password = new System.Security.SecureString();
//foreach (var c in "Yt1NyDpQNm".ToCharArray())
//{
// psexecProcess.StartInfo.Password.AppendChar(c);
//}
//**/
//Log.Debug("psexec args:\n" + psexecProcess.StartInfo.Arguments);
////psexecProcess.Start();
//Log.Debug("psexec process started");
//string execMessage = /*psexecProcess.StandardOutput.ReadToEnd() + " " +*/ "1 " + PS_PID_START_MSG + "5."; //psexecProcess.StandardError.ReadToEnd();
//execMessage = execMessage.Trim();
////psexecProcess.WaitForExit();
//System.Threading.Thread.Sleep(3000);
//Log.Debug("psexec output:\n" + execMessage);
//if (!execMessage.Contains(PS_PID_START_MSG))
// throw new Exception(String.Format(
// "Couldn't exec task {0} on win pc {1}",
// taskId, resParams.name
// ));
//execMessage = execMessage.Remove(0, execMessage.IndexOf(PS_PID_START_MSG) + PS_PID_START_MSG.Length);
//string pid = execMessage.Substring(0, execMessage.Length-1);
var rexService = EntryPointProxy.GetREx(node.Services.ExecutionUrl);
int pid = rexService.Exec(taskId);
Log.Debug(String.Format(
"Task {0} ({1}) started on pc {2} with pid = {3}",
taskId, pack.Name, node.NodeName, pid
));
_nodeUsed[node.NodeName] = true;
//System.Threading.Thread.Sleep(1000);
return pid + "\n" + node.NodeName;
}
}
protected override bool Matches(TaskDescription task, Resource resource)
{
/*
if (nodes.First().ClusterName.ToLower() == REQUIRED_CLUSTER_NAME.ToLower())
return true;
/**/
return false;
}
public static ServiceProxies.ControllerFarmService.ControllerFarmServiceClient GetControllerFarm(Resource resource)
{
return GetControllerFarm(resource.Controller.Url);
}