public Task(TaskDescription description)
: base(description)
{
if (Priority == TaskPriority.Urgent)
{
if (!ExecParams.ContainsKey("MinTime"))
ExecParams["MinTime"] = "0";
if (!ExecParams.ContainsKey("MaxTime"))
ExecParams["MaxTime"] = ExecParams["MinTime"];
}
OutputParams = new Dictionary<string, string>();
Time = new TaskTimeMeasurement();
Time.Started(TaskTimeMetric.Postponed);
if (this.IsFake())
Time.Edge(started: TaskTimeMetric.Queued, finished: TaskTimeMetric.Postponed);
Estimations = null;
CurrentSchedule = null;
Incarnation = new IncarnationParams();
State = TaskState.Defined;
string stepName = ExecParams.ContainsKey("StepName")? ExecParams["StepName"]: null;
string storageRoot = IOProxy.Storage.BuildPath(UserId, WfId, stepId: TaskId.ToString(), stepName: stepName);
this.PackageEngineState = new PackageEngineState(description, storageRoot); // todo : measure PackageEngineState expenses
}
public Task(Task otherTask)
: base(otherTask)
{
if (otherTask.OutputParams != null)
OutputParams = new Dictionary<string, string>(otherTask.OutputParams);
if (otherTask.PackageEngineState != null)
this.PackageEngineState = (PackageEngineState) otherTask.PackageEngineState.Clone();
Time = new TaskTimeMeasurement(otherTask.Time);
Incarnation = new IncarnationParams(otherTask.Incarnation);
if (otherTask.Estimations != null)
Estimations = new Dictionary<string, double>(otherTask.Estimations);
CurrentSchedule = null;
if (otherTask.CurrentSchedule != null)
{
CurrentSchedule = new TaskSchedule(otherTask.CurrentSchedule);
// immutable:
this.AssignedResource = otherTask.AssignedResource;
this.AssignedNodes = otherTask.AssignedNodes;
}
State = otherTask.State;
_inputsProcessed = otherTask._inputsProcessed;
_inputsProcessingError = otherTask._inputsProcessingError;
_failReason = otherTask._failReason;
if (otherTask._lastEvent != null && otherTask._lastEvent.HasValue)
_lastEvent = otherTask._lastEvent.Value;
}
private string GetFtpFolder(NodeConfig nodeConfig, Resource resource, CopyPhase phase)
{
string ftpFolder = resource.Nodes.First(n => n.NodeName == nodeConfig.NodeName).DataFolders.ExchangeUrlFromSystem;
string incarnatedFtpFolder = IncarnationParams.IncarnatePath(ftpFolder, TaskId, phase);
return(incarnatedFtpFolder);
}
public IncarnationParams(IncarnationParams otherIncarnation)
: this()
{
if (otherIncarnation != null)
{
this.CommandLine = otherIncarnation.CommandLine;
//this.StdInFile = otherIncarnation.StdInFile;
//this.StdOutFile = otherIncarnation.StdOutFile;
//this.PackageNameInConfig = otherIncarnation.PackageNameInConfig;
this.PackageName = otherIncarnation.PackageName;
//this.ProvidedTaskId = otherIncarnation.ProvidedTaskId;
if (otherIncarnation.FilesToCopy.Any())
{
FilesToCopy = otherIncarnation.FilesToCopy.Select(descr => descr).ToArray(); // todo : maybe Incarnation.FilesToCopy IEnumerable -> array[]?
}
if (otherIncarnation.ExpectedOutputFileNames.Any())
{
ExpectedOutputFileNames = otherIncarnation.ExpectedOutputFileNames.Select(name => name).ToArray(); // todo : maybe Incarnation.ExpectedOutputFileNames IEnumerable -> array[]?
}
CanExpectMoreFiles = otherIncarnation.CanExpectMoreFiles;
//this.InputFolderPath = otherIncarnation.InputFolderPath;
//this.OutputFolderPath = otherIncarnation.OutputFolderPath;
}
}
public IncarnationParams(IncarnationParams otherIncarnation)
: this()
{
if (otherIncarnation != null)
{
this.CommandLine = otherIncarnation.CommandLine;
//this.StdInFile = otherIncarnation.StdInFile;
//this.StdOutFile = otherIncarnation.StdOutFile;
//this.PackageNameInConfig = otherIncarnation.PackageNameInConfig;
this.PackageName = otherIncarnation.PackageName;
//this.ProvidedTaskId = otherIncarnation.ProvidedTaskId;
if (otherIncarnation.FilesToCopy.Any())
FilesToCopy = otherIncarnation.FilesToCopy.Select(descr => descr).ToArray(); // todo : maybe Incarnation.FilesToCopy IEnumerable -> array[]?
if (otherIncarnation.ExpectedOutputFileNames.Any())
ExpectedOutputFileNames = otherIncarnation.ExpectedOutputFileNames.Select(name => name).ToArray(); // todo : maybe Incarnation.ExpectedOutputFileNames IEnumerable -> array[]?
CanExpectMoreFiles = otherIncarnation.CanExpectMoreFiles;
//this.InputFolderPath = otherIncarnation.InputFolderPath;
//this.OutputFolderPath = otherIncarnation.OutputFolderPath;
}
}
public static IncarnationParams ProcessInputFiles(/* ref */ PackageEngineState engineState, out TimeSpan inputFilesTime)
{
var engine = new PackageEngine(engineState.CompiledDef, engineState.EngineContext);
var incarnation = new IncarnationParams();
var fileManager = new InputFileManager(engineState.StoragePathBase);
inputFilesTime = TimeSpanExt.Measure(() =>
{
TimeSpan timeLimit = PB_PROCESS_TIME_LIMIT;
try
{
Tools.ProcessWithTimeLimit(timeLimit, () =>
{
engine.PrepareInputs(fileManager);
CheckEngineResult(engine);
});
}
catch (TimeLimitException tle)
{
throw new PackageBaseException("Inputs processing in Package base exceeded time limit (" + timeLimit.TotalSeconds + " seconds).", tle);
}
finally
{
incarnation.FilesToCopy = incarnation.FilesToCopy.Concat(fileManager.GetInputFiles()).ToArray();
fileManager.Cleanup();
}
});
object cmdLine = engine.Ctx.Memory[CmdLineDef.DefName].Value;
//if (cmdLine == null)
//throw new PackageBaseException("CommandLine generated by PackageBase is empty");
incarnation.CommandLine = cmdLine.ToString();
Log.Debug(String.Format(
"Command line for pack {0}.{1} is '{2}' (task {3}). Cmdline object from PB is {4}",
engine.CompiledMode.ModeQName.PackageName ?? "''", engine.CompiledMode.ModeQName.ModeName ?? "''",
incarnation.CommandLine ?? "", engineState._taskDescription.TaskId,
cmdLine == null ? "null" : "not null"
));
if (String.IsNullOrWhiteSpace(incarnation.CommandLine))
{
throw new PackageBaseException("CommandLine generated by PackageBase is empty");
}
if (!engine.IsReadyToRun())
{
Log.Warn("PackageBase: task is not ready to run after PrepareInputs");
Log.Error("PackageBase: task is not ready to run after PrepareInputs");
throw new PackageBaseException("PackageBase: task is not ready to run after processing inputs");
}
return(incarnation);
}
public static IncarnationParams ProcessInputFiles(/* ref */ PackageEngineState engineState, out TimeSpan inputFilesTime)
{
var engine = new PackageEngine(engineState.CompiledDef, engineState.EngineContext);
var incarnation = new IncarnationParams();
var fileManager = new InputFileManager(engineState.StoragePathBase);
inputFilesTime = TimeSpanExt.Measure(() =>
{
TimeSpan timeLimit = PB_PROCESS_TIME_LIMIT;
try
{
Tools.ProcessWithTimeLimit(timeLimit, () =>
{
engine.PrepareInputs(fileManager);
CheckEngineResult(engine);
});
}
catch (TimeLimitException tle)
{
throw new PackageBaseException("Inputs processing in Package base exceeded time limit (" + timeLimit.TotalSeconds + " seconds).", tle);
}
finally
{
incarnation.FilesToCopy = incarnation.FilesToCopy.Concat(fileManager.GetInputFiles()).ToArray();
fileManager.Cleanup();
}
});
object cmdLine = engine.Ctx.Memory[CmdLineDef.DefName].Value;
//if (cmdLine == null)
//throw new PackageBaseException("CommandLine generated by PackageBase is empty");
incarnation.CommandLine = cmdLine.ToString();
Log.Debug(String.Format(
"Command line for pack {0}.{1} is '{2}' (task {3}). Cmdline object from PB is {4}",
engine.CompiledMode.ModeQName.PackageName ?? "''", engine.CompiledMode.ModeQName.ModeName ?? "''",
incarnation.CommandLine ?? "", engineState._taskDescription.TaskId,
cmdLine == null ? "null" : "not null"
));
if (String.IsNullOrWhiteSpace(incarnation.CommandLine))
throw new PackageBaseException("CommandLine generated by PackageBase is empty");
if (!engine.IsReadyToRun())
{
Log.Warn("PackageBase: task is not ready to run after PrepareInputs");
Log.Error("PackageBase: task is not ready to run after PrepareInputs");
throw new PackageBaseException("PackageBase: task is not ready to run after processing inputs");
}
return incarnation;
}
public override string Run(ulong taskId, IncarnationParams incarnation, Resource resource, IEnumerable <NodeConfig> nodesConfig)
{
lock (_lock)
{
int coresToUse = nodesConfig.Sum(conf => conf.Cores);
var node = GetDefaultNodeSettings(resource, nodesConfig);
var pack = node.PackageByName(incarnation.PackageName);
//string ftpFolder = IncarnationParams.IncarnatePath(node.DataFolders.ExchangeUrlFromSystem, taskId, CopyPhase.In);
//string jobFtpFolder = IncarnationParams.IncarnatePath(node.DataFolders.ExchangeUrlFromSystem, taskId, CopyPhase.None);
string ftpInputFolder = IncarnationParams.IncarnatePath(node.DataFolders.ExchangeUrlFromResource, taskId, CopyPhase.In);
string ftpOutputFolder = IncarnationParams.IncarnatePath(node.DataFolders.ExchangeUrlFromResource, taskId, CopyPhase.Out);
string clusterHomeFolder = IncarnationParams.IncarnatePath(node.DataFolders.LocalFolder, taskId, CopyPhase.None);
//IOProxy.Ftp.MakePath(ftpInputFolder);
//IOProxy.Ftp.MakePath(ftpOutputFolder);
try
{
SshExec(node, "mkdir " + clusterHomeFolder);
}
catch (Exception e)
{
Log.Warn(e.Message);
}
string fileNames = ""; //String.Join(" ", incarnation.FilesToCopy.Select(f => f.FileName));
foreach (var file in incarnation.FilesToCopy)
{
string tmpFile = System.IO.Path.GetTempFileName();
IOProxy.Storage.Download(file.StorageId, tmpFile);
string fileOnCluster = clusterHomeFolder.TrimEnd(new[] { '/', '\\' }) + "/" + file.FileName;
fileNames += " " + fileOnCluster;
ScpCopy(node, fileOnCluster, tmpFile);
System.IO.File.Delete(tmpFile);
}
string cmdLine = String.Format(incarnation.CommandLine, pack.AppPath, "clavire" + taskId.ToString(), fileNames.Trim());
string result = SshExec(node, cmdLine);
string jobId = result.Split(new[] { '\r', '\n' }, StringSplitOptions.RemoveEmptyEntries).First();
_nodeUsed[node.NodeName] = true;
return(jobId);
}
}
public Task(Task otherTask)
: base(otherTask)
{
if (otherTask.OutputParams != null)
{
OutputParams = new Dictionary <string, string>(otherTask.OutputParams);
}
if (otherTask.PackageEngineState != null)
{
this.PackageEngineState = (PackageEngineState)otherTask.PackageEngineState.Clone();
}
Time = new TaskTimeMeasurement(otherTask.Time);
Incarnation = new IncarnationParams(otherTask.Incarnation);
if (otherTask.Estimations != null)
{
Estimations = new Dictionary <string, double>(otherTask.Estimations);
}
CurrentSchedule = null;
if (otherTask.CurrentSchedule != null)
{
CurrentSchedule = new TaskSchedule(otherTask.CurrentSchedule);
// immutable:
this.AssignedResource = otherTask.AssignedResource;
this.AssignedNodes = otherTask.AssignedNodes;
}
State = otherTask.State;
_inputsProcessed = otherTask._inputsProcessed;
_inputsProcessingError = otherTask._inputsProcessingError;
_failReason = otherTask._failReason;
if (otherTask._lastEvent != null && otherTask._lastEvent.HasValue)
{
_lastEvent = otherTask._lastEvent.Value;
}
}
public override Tuple <TaskState, string> GetTaskState(ulong taskId, string providedTaskId, Resource resource, IEnumerable <NodeConfig> nodesConfig)
{
lock (_gridLock)
{
RefreshCertificate();
string taskIdString = taskId.ToString();
string state = SshExec(PilotCommands.JobStatus, providedTaskId).ToLower();
if (state.Contains("is new"))
{
return(Tuple.Create(TaskState.Started, state));
}
//return Tuple.Create(TaskState.Scheduled, state);
if (state.Contains("is running") || state.Contains("is starting"))
{
return(Tuple.Create(TaskState.Started, state));
}
var node = GetDefaultNodeSettings(resource, nodesConfig);
string ftpOutFolderFromSystem = IncarnationParams.IncarnatePath(node.DataFolders.ExchangeUrlFromSystem, taskId, CopyPhase.Out);
string ftpOutFolderFromResource = IncarnationParams.IncarnatePath(node.DataFolders.ExchangeUrlFromResource, taskId, CopyPhase.Out);
string gridFolder = IncarnationParams.IncarnatePath(node.DataFolders.LocalFolder, taskId, CopyPhase.None);
IOProxy.Ftp.MakePath(ftpOutFolderFromSystem);
SshExec(PilotCommands.CopyFilesToGridFtp, gridFolder + " " + ftpOutFolderFromResource);
_nodeUsed[node.NodeName] = false;
if (state.Contains("is finished"))
{
return(Tuple.Create(TaskState.Completed, state));
}
else
{
return(Tuple.Create(TaskState.Failed, state));
}
}
}
public override Tuple <TaskState, string> GetTaskState(ulong taskId, string providedTaskId, Resource resource, IEnumerable <NodeConfig> nodesConfig)
{
lock (_lock)
{
string result = SshExec(resource.Nodes.First(), SshCommands.GetTaskState, providedTaskId).ToLowerInvariant();
if (result.Contains("job_state = R") || result.Contains("job_state = Q"))
{
return(new Tuple <TaskState, string>(TaskState.Started, result));
}
// esle if (Aborted, Failed)
else
{
var node = GetDefaultNodeSettings(resource, nodesConfig);
string ftpOutFolderFromSystem = IncarnationParams.IncarnatePath(node.DataFolders.ExchangeUrlFromSystem, taskId, CopyPhase.Out);
string clusterFolder = "share/ANSYS_JOBS/clavire" + taskId.ToString() + "_0001/";
IOProxy.Ftp.MakePath(ftpOutFolderFromSystem);
var fileNames =
SshExec(node, SshCommands.Ls, clusterFolder)
.Split(new[] { ", " }, StringSplitOptions.RemoveEmptyEntries)
.Where(st => !st.Contains("/"))
.Select(st => st.Replace("*", "").Replace("|", ""))
.Where(st => !st.Contains(".rst") && !st.Contains(".err") && !st.Contains(".esav"));
foreach (string fileName in fileNames)
{
string tmpFile = System.IO.Path.GetTempFileName();
ScpGet(node, clusterFolder + fileName, tmpFile);
IOProxy.Ftp.UploadLocalFile(tmpFile, ftpOutFolderFromSystem, fileName);
System.IO.File.Delete(tmpFile);
}
_nodeUsed[node.NodeName] = false;
return(new Tuple <TaskState, string>(TaskState.Completed, result));
}
}
}
public Task(TaskDescription description)
: base(description)
{
if (Priority == TaskPriority.Urgent)
{
if (!ExecParams.ContainsKey("MinTime"))
{
ExecParams["MinTime"] = "0";
}
if (!ExecParams.ContainsKey("MaxTime"))
{
ExecParams["MaxTime"] = ExecParams["MinTime"];
}
}
OutputParams = new Dictionary <string, string>();
Time = new TaskTimeMeasurement();
Time.Started(TaskTimeMetric.Postponed);
if (this.IsFake())
{
Time.Edge(started: TaskTimeMetric.Queued, finished: TaskTimeMetric.Postponed);
}
Estimations = null;
CurrentSchedule = null;
Incarnation = new IncarnationParams();
State = TaskState.Defined;
string stepName = ExecParams.ContainsKey("StepName")? ExecParams["StepName"]: null;
string storageRoot = IOProxy.Storage.BuildPath(UserId, WfId, stepId: TaskId.ToString(), stepName: stepName);
this.PackageEngineState = new PackageEngineState(description, storageRoot); // todo : measure PackageEngineState expenses
}
public override string Run(ulong taskId, IncarnationParams incarnation, Resource resource, IEnumerable<NodeConfig> nodesConfig)
{
lock (_pcLock)
{
//AcceptPsToolsEula();
int coresToUse = nodesConfig.Sum(conf => conf.Cores);
var node = GetDefaultNodeSettings(resource, nodesConfig);
if (_nodeUsed[node.NodeName])
throw new Exception(String.Format("Could not run task {0} on node {1}: node used by another task", taskId, node.NodeName));
string ftpFolder = IncarnationParams.IncarnatePath(node.DataFolders.ExchangeUrlFromSystem, taskId, CopyPhase.In);
string jobFtpFolder = IncarnationParams.IncarnatePath(node.DataFolders.ExchangeUrlFromSystem, taskId, CopyPhase.None);
string sharedInputFolder = IncarnationParams.IncarnatePath(node.DataFolders.ExchangeUrlFromResource, taskId, CopyPhase.In);
string sharedOutputFolder = IncarnationParams.IncarnatePath(node.DataFolders.ExchangeUrlFromResource, taskId, CopyPhase.Out);
string tmpFolder = IncarnationParams.IncarnatePath(node.DataFolders.LocalFolder, taskId, CopyPhase.None);
IOProxy.Ftp.MakePath(ftpFolder);
IOProxy.Ftp.MakePath(jobFtpFolder);
string jobFileName = "job_" + taskId + ".cmd";
Log.Info(String.Format(
"Trying to exec task {0} on win PC {1}",
taskId, node.NodeName
));
var pack = node.Packages.First(p => String.Equals(p.Name, incarnation.PackageName, StringComparison.InvariantCultureIgnoreCase));
string batchContent = "";
batchContent += "mkdir " + tmpFolder.TrimEnd(new char[] { '/', '\\' }) + Environment.NewLine;
if (Path.IsPathRooted(tmpFolder)) // change drive if needed
batchContent += Path.GetPathRoot(tmpFolder).TrimEnd(new char[] { '/', '\\' }) + Environment.NewLine;
batchContent += String.Format(
@"cd {0}" + Environment.NewLine,
tmpFolder.TrimEnd(new char[] { '/', '\\' })
);
batchContent += "echo %time% > clavire_script_started" + Environment.NewLine;
foreach (string copyPath in pack.CopyOnStartup)
batchContent += String.Format(
@"xcopy {0} {1}\ /z /s /e /c /i /h /r /y" + Environment.NewLine,
copyPath.TrimEnd(new char[] { '/', '\\' }),
tmpFolder.TrimEnd(new char[] { '/', '\\' })
);
batchContent += String.Format(
//@"ping localhost -w 1000 -n 50" + Environment.NewLine +
@"xcopy {0} {1}\ /z /s /e /c /i /h /r /y" + Environment.NewLine,
sharedInputFolder.TrimEnd(new char[] {'/', '\\'}),
tmpFolder.TrimEnd(new char[] { '/', '\\' })
);
// todo : env vars on WinPc provider
string commandLine = incarnation.CommandLine;
//var pack = node.Packages.First(p => commandLine.StartsWith(p.Name, StringComparison.InvariantCultureIgnoreCase));
//commandLine = pack.Params["appPath"] + commandLine.Substring(pack.Name.Length);
commandLine = String.Format(incarnation.CommandLine, pack.AppPath);
//commandLine = String.Format(incarnation.CommandLine, pack.Params["appPath"]);
batchContent += "echo %time% > clavire_task_started" + Environment.NewLine;
batchContent += //"start \"" + jobFileName + " " + incarnation.PackageNameInConfig + "\" /wait /b" +
"cmd.exe /c " + commandLine + Environment.NewLine;
batchContent += "echo %time% > clavire_task_finished" + Environment.NewLine;
foreach (string delPath in pack.Cleanup)
{
batchContent += String.Format(
@"rmdir /s /q {0}" + Environment.NewLine +
@"del /f /s /q {0}" + Environment.NewLine,
tmpFolder + delPath
);
}
batchContent += String.Format(
@"xcopy {1} {0}\ /z /s /e /c /i /h /r /y" + Environment.NewLine,
sharedOutputFolder.TrimEnd(new char[] { '/', '\\' }),
tmpFolder.TrimEnd(new char[] { '/', '\\' })
);
batchContent += String.Format(
@"ping localhost -n 3" + Environment.NewLine +
@"echo %time% > clavire_script_finished" + Environment.NewLine +
@"xcopy clavire_script_finished {1}\ /z /s /e /c /i /h /r /y" + Environment.NewLine +
@"cd {0}" + Environment.NewLine +
@"cd .." + Environment.NewLine +
//@"rmdir /s /q {0}" + Environment.NewLine +
"",
tmpFolder.TrimEnd(new char[] { '/', '\\' }),
sharedOutputFolder.TrimEnd(new char[] { '/', '\\' })
);
IOProxy.Ftp.UploadFileContent(batchContent, jobFtpFolder, jobFileName);
//string cmdArgs = "/c " + CONST.Path.PsExec.Replace("PsExec.exe", "p.cmd");
//string cmdArgs = "\\\\192.168.4.1 -u nano -p Yt1NyDpQNm -d cmd.exe /c \"\\\\192.168.4.1\\ftp_exchange\\Tasks\\10043\\job_10043.cmd\"";
//Log.Debug(cmdArgs);
//Process.Start(CONST.Path.PsExec, cmdArgs);
//**/
//var psexecProcess = new Process();
//psexecProcess.StartInfo.UseShellExecute = false;
////psexecProcess.StartInfo.RedirectStandardOutput = true;
////psexecProcess.StartInfo.RedirectStandardError = true;
//psexecProcess.StartInfo.FileName = CONST.Path.PsExec;
//psexecProcess.StartInfo.Arguments = String.Format(
// "\\\\{0} -d -u {1} -p {2} cmd.exe /c {4}", // -d -w \"{3}\" ^> C:\\Temp\\out
// //"-u nano -p Yt1NyDpQNm cmd.exe /c " + CONST.Path.PsExec.Replace("PsExec.exe", "p.cmd"),
// resParams.name, resParams.user, resParams.pass,
// resParams.tempFolderOnMachine.Replace(@"\", @"\\"),
// sharedJobFilePath
//);
//*
//psexecProcess.StartInfo.UserName = "******";
//psexecProcess.StartInfo.Password = new System.Security.SecureString();
//foreach (var c in "Yt1NyDpQNm".ToCharArray())
//{
// psexecProcess.StartInfo.Password.AppendChar(c);
//}
//**/
//Log.Debug("psexec args:\n" + psexecProcess.StartInfo.Arguments);
////psexecProcess.Start();
//Log.Debug("psexec process started");
//string execMessage = /*psexecProcess.StandardOutput.ReadToEnd() + " " +*/ "1 " + PS_PID_START_MSG + "5."; //psexecProcess.StandardError.ReadToEnd();
//execMessage = execMessage.Trim();
////psexecProcess.WaitForExit();
//System.Threading.Thread.Sleep(3000);
//Log.Debug("psexec output:\n" + execMessage);
//if (!execMessage.Contains(PS_PID_START_MSG))
// throw new Exception(String.Format(
// "Couldn't exec task {0} on win pc {1}",
// taskId, resParams.name
// ));
//execMessage = execMessage.Remove(0, execMessage.IndexOf(PS_PID_START_MSG) + PS_PID_START_MSG.Length);
//string pid = execMessage.Substring(0, execMessage.Length-1);
var rexService = EntryPointProxy.GetREx(node.Services.ExecutionUrl);
int pid = rexService.Exec(taskId);
Log.Debug(String.Format(
"Task {0} ({1}) started on pc {2} with pid = {3}",
taskId, pack.Name, node.NodeName, pid
));
_nodeUsed[node.NodeName] = true;
//System.Threading.Thread.Sleep(1000);
return pid + "\n" + node.NodeName;
}
}
public override string Run(ulong taskId, IncarnationParams incarnation, Resource resource, IEnumerable<NodeConfig> nodesConfig)
{
string providedTaskId = taskId.ToString();
var node = GetDefaultNodeSettings(resource, nodesConfig);
var pack = node.PackageByName(incarnation.PackageName);
var service = EntryPointProxy.GetClustersService();
ClustersService.Code errCode;
ClustersService.TaskInfo taskInfo;
lock (_clustersServiceLock)
{
taskInfo = service.GetTaskState(providedTaskId, out errCode);
}
if (errCode != ServiceProxies.ClustersService.Code.OperationSuccess)
throw new ClusterException(errCode);
taskInfo.ClusterName = resource.ResourceName;
taskInfo.CommandLine = String.Format(incarnation.CommandLine, pack.AppPath);
taskInfo.PackageName = incarnation.PackageName.ToUpperInvariant();
/*
if (!String.IsNullOrEmpty(incarnation.StdInFile))
taskInfo.StdinFileName = incarnation.StdInFile;
else
taskInfo.StdinFileName = "";
if (!String.IsNullOrEmpty(incarnation.StdOutFile))
taskInfo.StdoutFileName = incarnation.StdOutFile;
else
taskInfo.StdoutFileName = "";
*/
// cores on nodes: {n, 0, 0} -> {n}
taskInfo.NumberOfCores = new ClustersService.ArrayOfInt();
taskInfo.NumberOfCores.AddRange(nodesConfig.Where(conf => conf.Cores > 0).Select(conf => conf.Cores));
taskInfo.NumberOfNodes = taskInfo.NumberOfCores.Count;
var logStream = new StringWriter();
logStream.WriteLine("Задача {0} ({1}) запускается на кластере {2}",
taskInfo.TaskID, taskInfo.PackageName, taskInfo.ClusterName);
logStream.WriteLine(" Папка с файлами расчета: {0}", taskInfo.FTPPath);
logStream.WriteLine(" Строка запуска: {0}", taskInfo.CommandLine);
logStream.WriteLine(" Перенаправление вывода: {0}", taskInfo.StdoutFileName);
logStream.Write(" Количество ядер (по каждому узлу): ");
foreach (int coresCount in taskInfo.NumberOfCores)
logStream.Write("{0} ", coresCount);
Log.Info(logStream.ToString());
lock (_clustersServiceLock)
{
errCode = service.ExecuteTask(taskInfo);
}
if (errCode != ServiceProxies.ClustersService.Code.OperationSuccess)
throw new ClusterException(String.Format(
CONST.Dirty<string>("Ошибка интегратора управления кластерами при запуске задачи: {0}"), errCode.ToString()
));
return providedTaskId;
}
public override string Run(ulong taskId, IncarnationParams incarnation, Resource resource, IEnumerable <NodeConfig> nodesConfig)
{
lock (_hadoopLock)
{
var node = GetDefaultNodeSettings(resource, nodesConfig);
var pack = node.PackageByName(incarnation.PackageName);
string workDir = IncarnationParams.IncarnatePath(node.DataFolders.LocalFolder, taskId, CopyPhase.None);
string workScriptPath = workDir + "hadoop.sh";
string workScriptInternalPath = workDir + "hadoop_internal.sh";
string exchangeInDir = IncarnationParams.IncarnatePath(node.DataFolders.ExchangeUrlFromResource, taskId, CopyPhase.In);
string exchangeOutDir = IncarnationParams.IncarnatePath(node.DataFolders.ExchangeUrlFromResource, taskId, CopyPhase.Out);
string identity = exchangeInDir.Split('@')[0].Replace("ftp://", "");
string user = identity.Split(':')[0];
string pass = identity.Split(':')[1];
string hostAndPath = exchangeInDir.Split('@')[1];
string ftpHost = hostAndPath.Split(new char[] { '/' }, 2)[0];
string ftpInPath = hostAndPath.Split(new char[] { '/' }, 2)[1];
string ftpOutPath = exchangeOutDir.Split('@')[1].Split(new char[] { '/' }, 2)[1];
//pack.Params.
var cmd = new StringBuilder();
cmd.AppendFormat("cd " + workDir + "\n");
cmd.AppendFormat(String.Format(HadoopCommands.Run, workScriptInternalPath) + "\n");
cmd.AppendFormat("echo $!" + "\n");
string tempPath = Path.GetTempFileName();
File.WriteAllText(tempPath, cmd.ToString());
var cmd_internal = new StringBuilder();
cmd_internal.AppendFormat("ftp -n {0} << END_SCRIPT\n", ftpHost);
cmd_internal.AppendFormat("quote User {0}\n", user);
cmd_internal.AppendFormat("quote PASS {0}\n", pass);
foreach (TaskFileDescription fileDesc in incarnation.FilesToCopy)
{
cmd_internal.AppendFormat("get {0}{2} {1}{2}\n", ftpInPath, workDir, fileDesc.FileName);
}
cmd_internal.AppendFormat("quit" + "\n");
cmd_internal.AppendFormat("END_SCRIPT" + "\n");
cmd_internal.AppendFormat(String.Format(incarnation.CommandLine, IncarnationParams.IncarnatePath(pack.AppPath, taskId, CopyPhase.None).TrimEnd('/')) + "\n");
cmd_internal.AppendFormat("ftp -n {0} << END_SCRIPT\n", ftpHost);
cmd_internal.AppendFormat("quote User {0}\n", user);
cmd_internal.AppendFormat("quote PASS {0}\n", pass);
cmd_internal.AppendFormat("mkdir {0}\n", ftpOutPath);
foreach (string fileName in incarnation.ExpectedOutputFileNames)
{
cmd_internal.AppendFormat("put {0}{2} {1}{2}\n", workDir, ftpOutPath, fileName);
}
cmd_internal.AppendFormat("quit" + "\n");
cmd_internal.AppendFormat("END_SCRIPT" + "\n");
string tempPathInternal = Path.GetTempFileName();
File.WriteAllText(tempPathInternal, cmd_internal.ToString());
string host = node.Services.ExecutionUrl;
SshRun(host, "mkdir " + workDir, true); // Need check, del
SshRun(host, "rm " + workDir + "*", true); // Need check
foreach (string path in pack.CopyOnStartup) // todo : internal_script maybe?
{
string toCopy = (path.EndsWith("/")? path + "*": path); // 'folder/*' needed for contents copy
// ^^^^^ doesn't work everywhere
SshRun(host, "cp -fpR " + toCopy + "/* " + workDir);
}
/*
* SshRun(host, "cp " + "/home/clavire/hadrawler/clavire.sh " + workDir); // Need check, del
* SshRun(host, "cp " + "/home/clavire/hadrawler/hadoop2.conf " + workDir); // Need check, del
*/
ScpPut(host, tempPath, workScriptPath);// Need del
SshRun(host, "chmod 700 " + workScriptPath);
ScpPut(host, tempPathInternal, workScriptInternalPath);// Need del
SshRun(host, "chmod 700 " + workScriptInternalPath);
File.Delete(tempPath);
File.Delete(tempPathInternal);
string pid = SshShell(host, workScriptPath); // проверка на то запустилось ли
_nodeUsed = true;
return(pid.Split('\n')[2].TrimEnd('\r'));
}
}
public object Run(TaskRunContext task)
{
ulong taskId = task.TaskId;
int coresToUse = (int)task.NodesConfig.Sum(cfg => cfg.Cores);
var node = GetNode(task);
string ftpFolder = IncarnationParams.IncarnatePath(node.DataFolders.ExchangeUrlFromSystem, taskId, CopyPhase.In);
string jobFtpFolder = IncarnationParams.IncarnatePath(node.DataFolders.ExchangeUrlFromSystem, taskId, CopyPhase.None);
string sharedInputFolder = IncarnationParams.IncarnatePath(node.DataFolders.ExchangeUrlFromResource, taskId, CopyPhase.In);
string sharedOutputFolder = IncarnationParams.IncarnatePath(node.DataFolders.ExchangeUrlFromResource, taskId, CopyPhase.Out);
string tmpFolder = IncarnationParams.IncarnatePath(node.DataFolders.LocalFolder, taskId, CopyPhase.None);
IOProxy.Ftp.MakePath(ftpFolder);
IOProxy.Ftp.MakePath(jobFtpFolder);
string jobFileName = "job_" + taskId + ".cmd";
logger.Info("Trying to exec task {0} on win PC {1}.{2}", taskId, node.ResourceName, node.NodeName);
var pack = node.Packages.First(p => String.Equals(p.Name, task.PackageName, StringComparison.InvariantCultureIgnoreCase));
string batchContent = "";
batchContent += "mkdir " + tmpFolder.TrimEnd(new char[] { '/', '\\' }) + Environment.NewLine;
if (Path.IsPathRooted(tmpFolder)) // change drive if needed
{
batchContent += Path.GetPathRoot(tmpFolder).TrimEnd(new char[] { '/', '\\' }) + Environment.NewLine;
}
batchContent += String.Format(
@"cd {0}" + Environment.NewLine,
tmpFolder.TrimEnd(new char[] { '/', '\\' })
);
batchContent += "echo %time% > clavire_script_started" + Environment.NewLine;
foreach (string copyPath in pack.CopyOnStartup)
{
batchContent += String.Format(
@"xcopy {0} {1}\ /z /s /e /c /i /h /r /y" + Environment.NewLine,
copyPath.TrimEnd(new char[] { '/', '\\' }),
tmpFolder.TrimEnd(new char[] { '/', '\\' })
);
}
batchContent += String.Format(
//@"ping localhost -w 1000 -n 50" + Environment.NewLine +
@"xcopy {0} {1}\ /z /s /e /c /i /h /r /y" + Environment.NewLine,
sharedInputFolder.TrimEnd(new char[] { '/', '\\' }),
tmpFolder.TrimEnd(new char[] { '/', '\\' })
);
foreach (var envVar in pack.EnvVars)
{
batchContent += "set " + envVar.Key + "=" + envVar.Value + Environment.NewLine;
}
string commandLine = task.CommandLine;
//var pack = node.Packages.First(p => commandLine.StartsWith(p.Name, StringComparison.InvariantCultureIgnoreCase));
//commandLine = pack.Params["appPath"] + commandLine.Substring(pack.Name.Length);
commandLine = String.Format(task.CommandLine, pack.AppPath);
//commandLine = String.Format(incarnation.CommandLine, pack.Params["appPath"]);
batchContent += "echo %time% > clavire_task_started" + Environment.NewLine;
batchContent += //"start \"" + jobFileName + " " + incarnation.PackageNameInConfig + "\" /wait /b" +
"cmd.exe /c " + commandLine + Environment.NewLine;
batchContent += "echo %time% > clavire_task_finished" + Environment.NewLine;
foreach (string copyPath in pack.CleanupIgnore)
{
batchContent += String.Format(
@"xcopy {1} {0} /z /s /e /c /i /h /r /y" + Environment.NewLine,
(sharedOutputFolder.TrimEnd(new char[] { '/', '\\' }) + "/" + copyPath.TrimStart(new char[] { '/', '\\' })).Replace("/", "\\"),
(tmpFolder.TrimEnd(new char[] { '/', '\\' }) + "/" + copyPath.TrimStart(new char[] { '/', '\\' })).Replace("/", "\\")
);
}
foreach (string delPath in pack.Cleanup)
{
batchContent += String.Format(
@"rmdir /s /q {0}" + Environment.NewLine +
@"del /f /s /q {0}" + Environment.NewLine,
tmpFolder + delPath // todo: delPath.TrimStart
);
}
batchContent += String.Format(
@"xcopy {1} {0}\ /z /s /e /c /i /h /r /y" + Environment.NewLine,
sharedOutputFolder.TrimEnd(new char[] { '/', '\\' }),
tmpFolder.TrimEnd(new char[] { '/', '\\' })
);
batchContent += String.Format(
//@"ping localhost -n 3" + Environment.NewLine +
@"echo %time% > clavire_script_finished" + Environment.NewLine +
@"xcopy clavire_script_finished {1}\ /z /s /e /c /i /h /r /y" + Environment.NewLine +
@"cd {0}" + Environment.NewLine +
@"cd .." + Environment.NewLine +
//@"rmdir /s /q {0}" + Environment.NewLine +
"",
tmpFolder.TrimEnd(new char[] { '/', '\\' }),
sharedOutputFolder.TrimEnd(new char[] { '/', '\\' })
);
int pauseLine = -1;
Int32.TryParse(Config.AppSettings[DEBUG_PAUSE_PARAM_NAME] ?? "-1", out pauseLine);
if (pauseLine >= 0)
{
var batchLines = batchContent.Replace("\r", "").Split(new[] { '\n' });
string newBatchContent =
String.Join(Environment.NewLine, batchLines.Take(pauseLine)) + Environment.NewLine +
"pause" + Environment.NewLine +
String.Join(Environment.NewLine, batchLines.Skip(pauseLine));
batchContent = newBatchContent;
}
IOProxy.Ftp.UploadFileContent(batchContent, jobFtpFolder, jobFileName);
var rexService = GetREx(node.Services.ExecutionUrl); // todo : close service client!
int pid = rexService.Exec(taskId);
logger.Info("Task {0} ({1}) started on pc {2}.{3} with pid = {4}", taskId, pack.Name, node.ResourceName, node.NodeName, pid);
return(pid + "\n" + node.NodeName);
}
/// <summary> /// Run task on selected resource /// </summary> /// <param name="incarnation"></param> /// <param name="resource"></param> /// <returns>Provided task id (unique for resource)</returns> public abstract string Run(ulong taskId, IncarnationParams incarnation, Resource resource, IEnumerable <NodeConfig> nodesConfig);
public override string Run(ulong taskId, IncarnationParams incarnation, Resource resource, IEnumerable <NodeConfig> nodesConfig)
{
lock (_gridLock)
{
RefreshCertificate();
string tmpFileName = null;
if (incarnation.UserCert != null)
{
Log.Info("Using user's certificate");
tmpFileName = Path.GetTempFileName();
IOProxy.Storage.Download(incarnation.UserCert, tmpFileName);
var scpForCert = new SSH.Scp(HELPER_SSH_HOST, HELPER_SSH_USER, HELPER_SSH_PASS);
scpForCert.Connect();
scpForCert.Recursive = true;
scpForCert.Put(tmpFileName, "/tmp/x509up_u500");
scpForCert.Close();
File.Delete(tmpFileName);
SshExec(PilotCommands.SetPermissionsOnProxyCertFile);
}
else
{
Log.Info("Using system's certificate");
}
try
{
int coresToUse = nodesConfig.Sum(conf => conf.Cores);
var node = GetDefaultNodeSettings(resource, nodesConfig);
var pack = node.PackageByName(incarnation.PackageName);
if (_nodeUsed[node.NodeName])
{
throw new Exception(String.Format("Could not run task {0} on node {1}: node used by another task", taskId, node.NodeName));
}
// todo : remove
incarnation.CommandLine = incarnation.CommandLine.Replace("java -jar ", "");
if (incarnation.PackageName.ToLowerInvariant() == "cnm")
{
incarnation.CommandLine = incarnation.CommandLine.Replace("{0}", "ru.ifmo.hpc.main.ExtendedModel");
}
else
if (incarnation.PackageName.ToLowerInvariant() == "ism")
{
incarnation.CommandLine = incarnation.CommandLine.Replace("{0}", "ru.ifmo.hpc.main.SpreadModel");
}
else
{
//if (incarnation.PackageName.ToLowerInvariant() == "orca")
incarnation.CommandLine = incarnation.CommandLine.Replace("{0}", "");
}
string ftpFolderFromSystem = IncarnationParams.IncarnatePath(node.DataFolders.ExchangeUrlFromSystem, taskId, CopyPhase.In);
string ftpFolderFromResource = IncarnationParams.IncarnatePath(node.DataFolders.ExchangeUrlFromResource, taskId, CopyPhase.In);
string gridFtpFolder = IncarnationParams.IncarnatePath(node.DataFolders.LocalFolder, taskId, CopyPhase.None);
SshExec(PilotCommands.MakeFolderOnGridFtp, gridFtpFolder);
string endl = "\n";
// Сначала дописываем недостающий входной файл (скрипт запуска пакета на кластере)
string scriptName = pack.AppPath;
//if (pack.EnvVars.Any())
{
// Файл с установкой переменных окружения, если пакет их использует
scriptName = "run.sh";
var scriptContent = new StringBuilder();
scriptContent.Append("#!/bin/bash" + endl);
foreach (var pair in pack.EnvVars)
{
scriptContent.AppendFormat("export {0}={1}" + endl, pair.Key, pair.Value);
}
scriptContent.Append(pack.AppPath);
if (incarnation.PackageName.ToLowerInvariant() == "orca")
{
string[] args = incarnation.CommandLine.Split(new[] { ' ' }, StringSplitOptions.RemoveEmptyEntries);
for (int i = 0; i < args.Length; i++)
{
if (args[i] == "orca.out")
{
scriptContent.Append(" >");
}
scriptContent.Append(" $" + (i + 1).ToString());
}
}
else
{
scriptContent.Append(" " + incarnation.CommandLine);
}
string scriptLocalPath = Path.GetTempFileName();
File.WriteAllText(scriptLocalPath, scriptContent.ToString());
IOProxy.Ftp.UploadLocalFile(scriptLocalPath, ftpFolderFromSystem, scriptName);
File.Delete(scriptLocalPath);
}
//IOProxy.Ftp.UploadLocalFile(DEFAULT_JOB_LAUNCHER_PATH, GetFtpInputFolder(taskId), Path.GetFileName(DEFAULT_JOB_LAUNCHER_PATH));
// Копируем входные файлы с ФТП на ГридФТП
SshExec(PilotCommands.CopyFilesToGridFtp, ftpFolderFromResource + " " + gridFtpFolder);
SshExec(PilotCommands.MakeFilesExecutableOnGridFtp, gridFtpFolder + "*");
// Формируем описание задания для грида
var jobFileContent = new StringBuilder();
jobFileContent.AppendFormat(@"{{ ""version"": 2, ""description"": ""{0}""," + endl, taskId);
jobFileContent.AppendFormat(@" ""default_storage_base"": ""{0}""," + endl, gridFtpFolder);
jobFileContent.AppendFormat(@" ""tasks"": [ {{ ""id"": ""a"", ""description"": ""task"", ""definition"": {{ ""version"": 2," + endl);
jobFileContent.AppendFormat(@" ""executable"": ""{0}""," + endl, scriptName);
//jobFileContent.AppendFormat(@" ""arguments"": [ ""{0}"" ]," + endl, String.Join(@""", """, args));
jobFileContent.AppendFormat(@" ""input_files"": {{" + endl);
if (scriptName == "run.sh") // todo : if no input files?
{
jobFileContent.AppendFormat(@" ""run.sh"": ""run.sh""," + endl);
}
jobFileContent.AppendFormat(@" " + String.Join(
"," + endl + " ",
incarnation.FilesToCopy.Select(
file => String.Format(@"""{0}"": ""{0}""", file.FileName)
)
));
jobFileContent.AppendFormat(endl + @" }}," + endl);
jobFileContent.AppendFormat(@" ""output_files"": {{" + endl);
//if (incarnation.PackageName.ToLowerInvariant() == "cnm")
// jobFileContent.AppendFormat(@" ""output.dat"": ""output.dat""" + endl);
//else
if (incarnation.PackageName.ToLowerInvariant() == "ism")
{
jobFileContent.AppendFormat(@" ""output.dat"": ""output.dat""" + endl);
}
else
if (incarnation.PackageName.ToLowerInvariant() == "orca")
{
jobFileContent.AppendFormat(@" ""orca.out"": ""orca.out""," + endl);
jobFileContent.AppendFormat(@" ""eldens.cube"": ""eldens.cube""" + endl);
}
else
{
jobFileContent.AppendFormat(@" " + String.Join(
"," + endl + " ",
incarnation.ExpectedOutputFileNames
.Where(name => name != "std.out" && name != "std.err")
.Select(
name => String.Format(@"""{0}"": ""{0}""", name)
)
) + endl);
}
jobFileContent.AppendFormat(@" }}," + endl);
jobFileContent.AppendFormat(@" ""stdout"": ""std.out"", ""stderr"": ""std.err"", " + endl);
jobFileContent.AppendFormat(@" ""count"": {0}" + endl, coresToUse);
if (pack.Params.ContainsKey("requirements"))
{
jobFileContent.AppendFormat(@" ,""requirements"": {0}" + endl, pack.Params["requirements"]);
}
jobFileContent.AppendFormat(@" }} }} ]," + endl);
jobFileContent.AppendFormat(@" ""requirements"": {{ ""hostname"": [""{0}""]", node.NodeAddress);
//if (pack.Params.ContainsKey("requirements"))
// jobFileContent.AppendFormat(@", {0}" + endl, pack.Params["requirements"]);
jobFileContent.AppendFormat(@"}}" + endl + "}}", node.NodeAddress);
Log.Debug(String.Format("Task's '{0}' grid job JSON: ", taskId, jobFileContent));
string jobFileName = "job_" + taskId.ToString() + ".js";
string jobFilePathOnHelper = JOBS_FOLDER_ON_HELPER + jobFileName;
//string jobFileContent = File.ReadAllText(DEFAULT_JOB_DESCR_PATH).Replace(GRIDFTP_PATH_TOKEN, taskFolderOnGridFtp);
string jobFilePathLocal = Path.GetTempFileName();
File.WriteAllText(jobFilePathLocal, jobFileContent.ToString());
// Записываем его на сервер с Пилотом
var scp = new SSH.Scp(HELPER_SSH_HOST, HELPER_SSH_USER, HELPER_SSH_PASS);
/*
* var notifier = new JobDescriptionUploadNotifier(TaskId, Cluster, RunParams);
* scp.OnTransferEnd += new SSH.FileTransferEvent(notifier.OnFinish); // todo : необязательно
*/
scp.Connect();
scp.Recursive = true;
scp.Put(jobFilePathLocal, jobFilePathOnHelper);
scp.Close();
File.Delete(jobFilePathLocal); // todo : remove files on helper and gridftp
// Запускаем
Log.Info(String.Format(
"Trying to exec task {0} on grid cluster {1}",
taskId, node.NodeName
));
string launchResult = SshExec(PilotCommands.SubmitJob, jobFilePathOnHelper, pilotUrl: node.Services.ExecutionUrl);
int urlPos = launchResult.IndexOf("https://");
string jobUrl = launchResult.Substring(urlPos).Trim() + "a";
Log.Debug(jobUrl);
Log.Info(String.Format(
"Task {0} launched on grid with jobUrl = {1}",
taskId, jobUrl
));
_nodeUsed[node.NodeName] = true;
return(jobUrl);
}
catch (Exception e)
{
Log.Error(String.Format(
"Error while starting task {0} in grid: {1}\n{2}",
taskId, e.Message, e.StackTrace
));
throw;
}
finally
{
if (incarnation.UserCert != null)
{
Log.Info("Wiping user's certificate");
tmpFileName = Path.GetTempFileName();
File.WriteAllText(tmpFileName, "Wiped by Easis system");
var scpForCert = new SSH.Scp(HELPER_SSH_HOST, HELPER_SSH_USER, HELPER_SSH_PASS);
scpForCert.Connect();
scpForCert.Recursive = true;
scpForCert.Put(tmpFileName, "/tmp/x509up_u500");
scpForCert.Close();
File.Delete(tmpFileName);
SshExec(PilotCommands.SetPermissionsOnProxyCertFile);
}
}
}
}
public override string Run(ulong taskId, IncarnationParams incarnation, Resource resource, IEnumerable<NodeConfig> nodesConfig)
{
lock (_gridLock)
{
RefreshCertificate();
string tmpFileName = null;
if (incarnation.UserCert != null)
{
Log.Info("Using user's certificate");
tmpFileName = Path.GetTempFileName();
IOProxy.Storage.Download(incarnation.UserCert, tmpFileName);
var scpForCert = new SSH.Scp(HELPER_SSH_HOST, HELPER_SSH_USER, HELPER_SSH_PASS);
scpForCert.Connect();
scpForCert.Recursive = true;
scpForCert.Put(tmpFileName, "/tmp/x509up_u500");
scpForCert.Close();
File.Delete(tmpFileName);
SshExec(PilotCommands.SetPermissionsOnProxyCertFile);
}
else
{
Log.Info("Using system's certificate");
}
try
{
int coresToUse = nodesConfig.Sum(conf => conf.Cores);
var node = GetDefaultNodeSettings(resource, nodesConfig);
var pack = node.PackageByName(incarnation.PackageName);
if (_nodeUsed[node.NodeName])
throw new Exception(String.Format("Could not run task {0} on node {1}: node used by another task", taskId, node.NodeName));
// todo : remove
incarnation.CommandLine = incarnation.CommandLine.Replace("java -jar ", "");
if (incarnation.PackageName.ToLowerInvariant() == "cnm")
incarnation.CommandLine = incarnation.CommandLine.Replace("{0}", "ru.ifmo.hpc.main.ExtendedModel");
else
if (incarnation.PackageName.ToLowerInvariant() == "ism")
incarnation.CommandLine = incarnation.CommandLine.Replace("{0}", "ru.ifmo.hpc.main.SpreadModel");
else
//if (incarnation.PackageName.ToLowerInvariant() == "orca")
incarnation.CommandLine = incarnation.CommandLine.Replace("{0}", "");
string ftpFolderFromSystem = IncarnationParams.IncarnatePath(node.DataFolders.ExchangeUrlFromSystem, taskId, CopyPhase.In);
string ftpFolderFromResource = IncarnationParams.IncarnatePath(node.DataFolders.ExchangeUrlFromResource, taskId, CopyPhase.In);
string gridFtpFolder = IncarnationParams.IncarnatePath(node.DataFolders.LocalFolder, taskId, CopyPhase.None);
SshExec(PilotCommands.MakeFolderOnGridFtp, gridFtpFolder);
string endl = "\n";
// Сначала дописываем недостающий входной файл (скрипт запуска пакета на кластере)
string scriptName = pack.AppPath;
//if (pack.EnvVars.Any())
{
// Файл с установкой переменных окружения, если пакет их использует
scriptName = "run.sh";
var scriptContent = new StringBuilder();
scriptContent.Append("#!/bin/bash" + endl);
foreach (var pair in pack.EnvVars)
scriptContent.AppendFormat("export {0}={1}" + endl, pair.Key, pair.Value);
scriptContent.Append(pack.AppPath);
if (incarnation.PackageName.ToLowerInvariant() == "orca")
{
string[] args = incarnation.CommandLine.Split(new[] { ' ' }, StringSplitOptions.RemoveEmptyEntries);
for (int i = 0; i < args.Length; i++)
{
if (args[i] == "orca.out")
scriptContent.Append(" >");
scriptContent.Append(" $" + (i + 1).ToString());
}
}
else
{
scriptContent.Append(" " + incarnation.CommandLine);
}
string scriptLocalPath = Path.GetTempFileName();
File.WriteAllText(scriptLocalPath, scriptContent.ToString());
IOProxy.Ftp.UploadLocalFile(scriptLocalPath, ftpFolderFromSystem, scriptName);
File.Delete(scriptLocalPath);
}
//IOProxy.Ftp.UploadLocalFile(DEFAULT_JOB_LAUNCHER_PATH, GetFtpInputFolder(taskId), Path.GetFileName(DEFAULT_JOB_LAUNCHER_PATH));
// Копируем входные файлы с ФТП на ГридФТП
SshExec(PilotCommands.CopyFilesToGridFtp, ftpFolderFromResource + " " + gridFtpFolder);
SshExec(PilotCommands.MakeFilesExecutableOnGridFtp, gridFtpFolder + "*");
// Формируем описание задания для грида
var jobFileContent = new StringBuilder();
jobFileContent.AppendFormat(@"{{ ""version"": 2, ""description"": ""{0}""," + endl, taskId);
jobFileContent.AppendFormat(@" ""default_storage_base"": ""{0}""," + endl, gridFtpFolder);
jobFileContent.AppendFormat(@" ""tasks"": [ {{ ""id"": ""a"", ""description"": ""task"", ""definition"": {{ ""version"": 2," + endl);
jobFileContent.AppendFormat(@" ""executable"": ""{0}""," + endl, scriptName);
//jobFileContent.AppendFormat(@" ""arguments"": [ ""{0}"" ]," + endl, String.Join(@""", """, args));
jobFileContent.AppendFormat(@" ""input_files"": {{" + endl);
if (scriptName == "run.sh") // todo : if no input files?
jobFileContent.AppendFormat(@" ""run.sh"": ""run.sh""," + endl);
jobFileContent.AppendFormat(@" " + String.Join(
"," + endl + " ",
incarnation.FilesToCopy.Select(
file => String.Format(@"""{0}"": ""{0}""", file.FileName)
)
));
jobFileContent.AppendFormat(endl + @" }}," + endl);
jobFileContent.AppendFormat(@" ""output_files"": {{" + endl);
//if (incarnation.PackageName.ToLowerInvariant() == "cnm")
// jobFileContent.AppendFormat(@" ""output.dat"": ""output.dat""" + endl);
//else
if (incarnation.PackageName.ToLowerInvariant() == "ism")
jobFileContent.AppendFormat(@" ""output.dat"": ""output.dat""" + endl);
else
if (incarnation.PackageName.ToLowerInvariant() == "orca")
{
jobFileContent.AppendFormat(@" ""orca.out"": ""orca.out""," + endl);
jobFileContent.AppendFormat(@" ""eldens.cube"": ""eldens.cube""" + endl);
}
else
{
jobFileContent.AppendFormat(@" " + String.Join(
"," + endl + " ",
incarnation.ExpectedOutputFileNames
.Where(name => name != "std.out" && name != "std.err")
.Select(
name => String.Format(@"""{0}"": ""{0}""", name)
)
) + endl);
}
jobFileContent.AppendFormat(@" }}," + endl);
jobFileContent.AppendFormat(@" ""stdout"": ""std.out"", ""stderr"": ""std.err"", " + endl);
jobFileContent.AppendFormat(@" ""count"": {0}" + endl, coresToUse);
if (pack.Params.ContainsKey("requirements"))
jobFileContent.AppendFormat(@" ,""requirements"": {0}" + endl, pack.Params["requirements"]);
jobFileContent.AppendFormat(@" }} }} ]," + endl);
jobFileContent.AppendFormat(@" ""requirements"": {{ ""hostname"": [""{0}""]", node.NodeAddress);
//if (pack.Params.ContainsKey("requirements"))
// jobFileContent.AppendFormat(@", {0}" + endl, pack.Params["requirements"]);
jobFileContent.AppendFormat(@"}}" + endl + "}}", node.NodeAddress);
Log.Debug(String.Format("Task's '{0}' grid job JSON: ", taskId, jobFileContent));
string jobFileName = "job_" + taskId.ToString() + ".js";
string jobFilePathOnHelper = JOBS_FOLDER_ON_HELPER + jobFileName;
//string jobFileContent = File.ReadAllText(DEFAULT_JOB_DESCR_PATH).Replace(GRIDFTP_PATH_TOKEN, taskFolderOnGridFtp);
string jobFilePathLocal = Path.GetTempFileName();
File.WriteAllText(jobFilePathLocal, jobFileContent.ToString());
// Записываем его на сервер с Пилотом
var scp = new SSH.Scp(HELPER_SSH_HOST, HELPER_SSH_USER, HELPER_SSH_PASS);
/*
var notifier = new JobDescriptionUploadNotifier(TaskId, Cluster, RunParams);
scp.OnTransferEnd += new SSH.FileTransferEvent(notifier.OnFinish); // todo : необязательно
*/
scp.Connect();
scp.Recursive = true;
scp.Put(jobFilePathLocal, jobFilePathOnHelper);
scp.Close();
File.Delete(jobFilePathLocal); // todo : remove files on helper and gridftp
// Запускаем
Log.Info(String.Format(
"Trying to exec task {0} on grid cluster {1}",
taskId, node.NodeName
));
string launchResult = SshExec(PilotCommands.SubmitJob, jobFilePathOnHelper, pilotUrl: node.Services.ExecutionUrl);
int urlPos = launchResult.IndexOf("https://");
string jobUrl = launchResult.Substring(urlPos).Trim() + "a";
Log.Debug(jobUrl);
Log.Info(String.Format(
"Task {0} launched on grid with jobUrl = {1}",
taskId, jobUrl
));
_nodeUsed[node.NodeName] = true;
return jobUrl;
}
catch (Exception e)
{
Log.Error(String.Format(
"Error while starting task {0} in grid: {1}\n{2}",
taskId, e.Message, e.StackTrace
));
throw;
}
finally
{
if (incarnation.UserCert != null)
{
Log.Info("Wiping user's certificate");
tmpFileName = Path.GetTempFileName();
File.WriteAllText(tmpFileName, "Wiped by Easis system");
var scpForCert = new SSH.Scp(HELPER_SSH_HOST, HELPER_SSH_USER, HELPER_SSH_PASS);
scpForCert.Connect();
scpForCert.Recursive = true;
scpForCert.Put(tmpFileName, "/tmp/x509up_u500");
scpForCert.Close();
File.Delete(tmpFileName);
SshExec(PilotCommands.SetPermissionsOnProxyCertFile);
}
}
}
}
/// <summary> /// Run task on selected resource /// </summary> /// <param name="incarnation"></param> /// <param name="resource"></param> /// <returns>Provided task id (unique for resource)</returns> public abstract string Run(ulong taskId, IncarnationParams incarnation, Resource resource, IEnumerable<NodeConfig> nodesConfig);
public override string Run(ulong taskId, IncarnationParams incarnation, Resource resource, IEnumerable <NodeConfig> nodesConfig)
{
string providedTaskId = taskId.ToString();
var node = GetDefaultNodeSettings(resource, nodesConfig);
var pack = node.PackageByName(incarnation.PackageName);
var service = EntryPointProxy.GetClustersService();
ClustersService.Code errCode;
ClustersService.TaskInfo taskInfo;
lock (_clustersServiceLock)
{
taskInfo = service.GetTaskState(providedTaskId, out errCode);
}
if (errCode != ServiceProxies.ClustersService.Code.OperationSuccess)
{
throw new ClusterException(errCode);
}
taskInfo.ClusterName = resource.ResourceName;
taskInfo.CommandLine = String.Format(incarnation.CommandLine, pack.AppPath);
taskInfo.PackageName = incarnation.PackageName.ToUpperInvariant();
/*
* if (!String.IsNullOrEmpty(incarnation.StdInFile))
* taskInfo.StdinFileName = incarnation.StdInFile;
* else
* taskInfo.StdinFileName = "";
*
* if (!String.IsNullOrEmpty(incarnation.StdOutFile))
* taskInfo.StdoutFileName = incarnation.StdOutFile;
* else
* taskInfo.StdoutFileName = "";
*/
// cores on nodes: {n, 0, 0} -> {n}
taskInfo.NumberOfCores = new ClustersService.ArrayOfInt();
taskInfo.NumberOfCores.AddRange(nodesConfig.Where(conf => conf.Cores > 0).Select(conf => conf.Cores));
taskInfo.NumberOfNodes = taskInfo.NumberOfCores.Count;
var logStream = new StringWriter();
logStream.WriteLine("Задача {0} ({1}) запускается на кластере {2}",
taskInfo.TaskID, taskInfo.PackageName, taskInfo.ClusterName);
logStream.WriteLine(" Папка с файлами расчета: {0}", taskInfo.FTPPath);
logStream.WriteLine(" Строка запуска: {0}", taskInfo.CommandLine);
logStream.WriteLine(" Перенаправление вывода: {0}", taskInfo.StdoutFileName);
logStream.Write(" Количество ядер (по каждому узлу): ");
foreach (int coresCount in taskInfo.NumberOfCores)
{
logStream.Write("{0} ", coresCount);
}
Log.Info(logStream.ToString());
lock (_clustersServiceLock)
{
errCode = service.ExecuteTask(taskInfo);
}
if (errCode != ServiceProxies.ClustersService.Code.OperationSuccess)
{
throw new ClusterException(String.Format(
CONST.Dirty <string>("Ошибка интегратора управления кластерами при запуске задачи: {0}"), errCode.ToString()
));
}
return(providedTaskId);
}
public override string Run(ulong taskId, IncarnationParams incarnation, Resource resource, IEnumerable<NodeConfig> nodesConfig)
{
lock (_hadoopLock)
{
var node = GetDefaultNodeSettings(resource, nodesConfig);
var pack = node.PackageByName(incarnation.PackageName);
string workDir = IncarnationParams.IncarnatePath(node.DataFolders.LocalFolder, taskId, CopyPhase.None);
string workScriptPath = workDir + "hadoop.sh";
string workScriptInternalPath = workDir + "hadoop_internal.sh";
string exchangeInDir = IncarnationParams.IncarnatePath(node.DataFolders.ExchangeUrlFromResource, taskId, CopyPhase.In);
string exchangeOutDir = IncarnationParams.IncarnatePath(node.DataFolders.ExchangeUrlFromResource, taskId, CopyPhase.Out);
string identity = exchangeInDir.Split('@')[0].Replace("ftp://", "");
string user = identity.Split(':')[0];
string pass = identity.Split(':')[1];
string hostAndPath = exchangeInDir.Split('@')[1];
string ftpHost = hostAndPath.Split(new char[] { '/' }, 2)[0];
string ftpInPath = hostAndPath.Split(new char[] { '/' }, 2)[1];
string ftpOutPath = exchangeOutDir.Split('@')[1].Split(new char[] { '/' }, 2)[1];
//pack.Params.
var cmd = new StringBuilder();
cmd.AppendFormat("cd " + workDir + "\n");
cmd.AppendFormat(String.Format(HadoopCommands.Run, workScriptInternalPath) + "\n");
cmd.AppendFormat("echo $!" + "\n");
string tempPath = Path.GetTempFileName();
File.WriteAllText(tempPath, cmd.ToString());
var cmd_internal = new StringBuilder();
cmd_internal.AppendFormat("ftp -n {0} << END_SCRIPT\n", ftpHost);
cmd_internal.AppendFormat("quote User {0}\n", user);
cmd_internal.AppendFormat("quote PASS {0}\n", pass);
foreach (TaskFileDescription fileDesc in incarnation.FilesToCopy)
{
cmd_internal.AppendFormat("get {0}{2} {1}{2}\n", ftpInPath, workDir, fileDesc.FileName);
}
cmd_internal.AppendFormat("quit" + "\n");
cmd_internal.AppendFormat("END_SCRIPT" + "\n");
cmd_internal.AppendFormat(String.Format(incarnation.CommandLine, IncarnationParams.IncarnatePath(pack.AppPath, taskId, CopyPhase.None).TrimEnd('/')) + "\n");
cmd_internal.AppendFormat("ftp -n {0} << END_SCRIPT\n", ftpHost);
cmd_internal.AppendFormat("quote User {0}\n", user);
cmd_internal.AppendFormat("quote PASS {0}\n", pass);
cmd_internal.AppendFormat("mkdir {0}\n", ftpOutPath);
foreach (string fileName in incarnation.ExpectedOutputFileNames)
{
cmd_internal.AppendFormat("put {0}{2} {1}{2}\n", workDir, ftpOutPath, fileName);
}
cmd_internal.AppendFormat("quit" + "\n");
cmd_internal.AppendFormat("END_SCRIPT" + "\n");
string tempPathInternal = Path.GetTempFileName();
File.WriteAllText(tempPathInternal, cmd_internal.ToString());
string host = node.Services.ExecutionUrl;
SshRun(host, "mkdir " + workDir, true); // Need check, del
SshRun(host, "rm " + workDir + "*", true); // Need check
foreach (string path in pack.CopyOnStartup) // todo : internal_script maybe?
{
string toCopy = (path.EndsWith("/")? path + "*": path); // 'folder/*' needed for contents copy
// ^^^^^ doesn't work everywhere
SshRun(host, "cp -fpR " + toCopy + "/* " + workDir);
}
/*
SshRun(host, "cp " + "/home/clavire/hadrawler/clavire.sh " + workDir); // Need check, del
SshRun(host, "cp " + "/home/clavire/hadrawler/hadoop2.conf " + workDir); // Need check, del
*/
ScpPut(host, tempPath, workScriptPath);// Need del
SshRun(host, "chmod 700 " + workScriptPath);
ScpPut(host, tempPathInternal, workScriptInternalPath);// Need del
SshRun(host, "chmod 700 " + workScriptInternalPath);
File.Delete(tempPath);
File.Delete(tempPathInternal);
string pid = SshShell(host, workScriptPath); // проверка на то запустилось ли
_nodeUsed = true;
return pid.Split('\n')[2].TrimEnd('\r');
}
}
public override string Run(ulong taskId, IncarnationParams incarnation, Resource resource, IEnumerable <NodeConfig> nodesConfig)
{
lock (_pcLock)
{
//AcceptPsToolsEula();
int coresToUse = nodesConfig.Sum(conf => conf.Cores);
var node = GetDefaultNodeSettings(resource, nodesConfig);
if (_nodeUsed[node.NodeName])
{
throw new Exception(String.Format("Could not run task {0} on node {1}: node used by another task", taskId, node.NodeName));
}
string ftpFolder = IncarnationParams.IncarnatePath(node.DataFolders.ExchangeUrlFromSystem, taskId, CopyPhase.In);
string jobFtpFolder = IncarnationParams.IncarnatePath(node.DataFolders.ExchangeUrlFromSystem, taskId, CopyPhase.None);
string sharedInputFolder = IncarnationParams.IncarnatePath(node.DataFolders.ExchangeUrlFromResource, taskId, CopyPhase.In);
string sharedOutputFolder = IncarnationParams.IncarnatePath(node.DataFolders.ExchangeUrlFromResource, taskId, CopyPhase.Out);
string tmpFolder = IncarnationParams.IncarnatePath(node.DataFolders.LocalFolder, taskId, CopyPhase.None);
IOProxy.Ftp.MakePath(ftpFolder);
IOProxy.Ftp.MakePath(jobFtpFolder);
string jobFileName = "job_" + taskId + ".cmd";
Log.Info(String.Format(
"Trying to exec task {0} on win PC {1}",
taskId, node.NodeName
));
var pack = node.Packages.First(p => String.Equals(p.Name, incarnation.PackageName, StringComparison.InvariantCultureIgnoreCase));
string batchContent = "";
batchContent += "mkdir " + tmpFolder.TrimEnd(new char[] { '/', '\\' }) + Environment.NewLine;
if (Path.IsPathRooted(tmpFolder)) // change drive if needed
{
batchContent += Path.GetPathRoot(tmpFolder).TrimEnd(new char[] { '/', '\\' }) + Environment.NewLine;
}
batchContent += String.Format(
@"cd {0}" + Environment.NewLine,
tmpFolder.TrimEnd(new char[] { '/', '\\' })
);
batchContent += "echo %time% > clavire_script_started" + Environment.NewLine;
foreach (string copyPath in pack.CopyOnStartup)
{
batchContent += String.Format(
@"xcopy {0} {1}\ /z /s /e /c /i /h /r /y" + Environment.NewLine,
copyPath.TrimEnd(new char[] { '/', '\\' }),
tmpFolder.TrimEnd(new char[] { '/', '\\' })
);
}
batchContent += String.Format(
//@"ping localhost -w 1000 -n 50" + Environment.NewLine +
@"xcopy {0} {1}\ /z /s /e /c /i /h /r /y" + Environment.NewLine,
sharedInputFolder.TrimEnd(new char[] { '/', '\\' }),
tmpFolder.TrimEnd(new char[] { '/', '\\' })
);
// todo : env vars on WinPc provider
string commandLine = incarnation.CommandLine;
//var pack = node.Packages.First(p => commandLine.StartsWith(p.Name, StringComparison.InvariantCultureIgnoreCase));
//commandLine = pack.Params["appPath"] + commandLine.Substring(pack.Name.Length);
commandLine = String.Format(incarnation.CommandLine, pack.AppPath);
//commandLine = String.Format(incarnation.CommandLine, pack.Params["appPath"]);
batchContent += "echo %time% > clavire_task_started" + Environment.NewLine;
batchContent += //"start \"" + jobFileName + " " + incarnation.PackageNameInConfig + "\" /wait /b" +
"cmd.exe /c " + commandLine + Environment.NewLine;
batchContent += "echo %time% > clavire_task_finished" + Environment.NewLine;
foreach (string delPath in pack.Cleanup)
{
batchContent += String.Format(
@"rmdir /s /q {0}" + Environment.NewLine +
@"del /f /s /q {0}" + Environment.NewLine,
tmpFolder + delPath
);
}
batchContent += String.Format(
@"xcopy {1} {0}\ /z /s /e /c /i /h /r /y" + Environment.NewLine,
sharedOutputFolder.TrimEnd(new char[] { '/', '\\' }),
tmpFolder.TrimEnd(new char[] { '/', '\\' })
);
batchContent += String.Format(
@"ping localhost -n 3" + Environment.NewLine +
@"echo %time% > clavire_script_finished" + Environment.NewLine +
@"xcopy clavire_script_finished {1}\ /z /s /e /c /i /h /r /y" + Environment.NewLine +
@"cd {0}" + Environment.NewLine +
@"cd .." + Environment.NewLine +
//@"rmdir /s /q {0}" + Environment.NewLine +
"",
tmpFolder.TrimEnd(new char[] { '/', '\\' }),
sharedOutputFolder.TrimEnd(new char[] { '/', '\\' })
);
IOProxy.Ftp.UploadFileContent(batchContent, jobFtpFolder, jobFileName);
//string cmdArgs = "/c " + CONST.Path.PsExec.Replace("PsExec.exe", "p.cmd");
//string cmdArgs = "\\\\192.168.4.1 -u nano -p Yt1NyDpQNm -d cmd.exe /c \"\\\\192.168.4.1\\ftp_exchange\\Tasks\\10043\\job_10043.cmd\"";
//Log.Debug(cmdArgs);
//Process.Start(CONST.Path.PsExec, cmdArgs);
//**/
//var psexecProcess = new Process();
//psexecProcess.StartInfo.UseShellExecute = false;
////psexecProcess.StartInfo.RedirectStandardOutput = true;
////psexecProcess.StartInfo.RedirectStandardError = true;
//psexecProcess.StartInfo.FileName = CONST.Path.PsExec;
//psexecProcess.StartInfo.Arguments = String.Format(
// "\\\\{0} -d -u {1} -p {2} cmd.exe /c {4}", // -d -w \"{3}\" ^> C:\\Temp\\out
// //"-u nano -p Yt1NyDpQNm cmd.exe /c " + CONST.Path.PsExec.Replace("PsExec.exe", "p.cmd"),
// resParams.name, resParams.user, resParams.pass,
// resParams.tempFolderOnMachine.Replace(@"\", @"\\"),
// sharedJobFilePath
//);
//*
//psexecProcess.StartInfo.UserName = "******";
//psexecProcess.StartInfo.Password = new System.Security.SecureString();
//foreach (var c in "Yt1NyDpQNm".ToCharArray())
//{
// psexecProcess.StartInfo.Password.AppendChar(c);
//}
//**/
//Log.Debug("psexec args:\n" + psexecProcess.StartInfo.Arguments);
////psexecProcess.Start();
//Log.Debug("psexec process started");
//string execMessage = /*psexecProcess.StandardOutput.ReadToEnd() + " " +*/ "1 " + PS_PID_START_MSG + "5."; //psexecProcess.StandardError.ReadToEnd();
//execMessage = execMessage.Trim();
////psexecProcess.WaitForExit();
//System.Threading.Thread.Sleep(3000);
//Log.Debug("psexec output:\n" + execMessage);
//if (!execMessage.Contains(PS_PID_START_MSG))
// throw new Exception(String.Format(
// "Couldn't exec task {0} on win pc {1}",
// taskId, resParams.name
// ));
//execMessage = execMessage.Remove(0, execMessage.IndexOf(PS_PID_START_MSG) + PS_PID_START_MSG.Length);
//string pid = execMessage.Substring(0, execMessage.Length-1);
var rexService = EntryPointProxy.GetREx(node.Services.ExecutionUrl);
int pid = rexService.Exec(taskId);
Log.Debug(String.Format(
"Task {0} ({1}) started on pc {2} with pid = {3}",
taskId, pack.Name, node.NodeName, pid
));
_nodeUsed[node.NodeName] = true;
//System.Threading.Thread.Sleep(1000);
return(pid + "\n" + node.NodeName);
}
}
