protected void PutFile(string host, NodeAuth auth, string local, string remote)
{
Scp scp = new Scp (host, auth.UserName);
SetupSSH (scp, auth);
scp.Put (local, remote);
scp.Close ();
}
private void CopyScript(string host, NodeAuth auth)
{
Scp scp = new Scp (host, auth.UserName);
SetupSSH (scp, auth);
scp.Put (LocalScriptPath, RemoteScriptPath);
scp.Close ();
}
protected void PutDirectory(string host, NodeAuth auth, string local, string remote)
{
Scp scp = new Scp (host, auth.UserName);
SetupSSH (scp, auth);
scp.To (local, remote, true);
scp.Close ();
}
private void ScpCopy(ResourceNode node, string remotePath, string localPath)
{
var scp = new Ssh.Scp(node.NodeAddress, node.Credentials.Username, node.Credentials.Password);
scp.Connect();
scp.Recursive = true;
scp.Put(localPath, remotePath);
scp.Close();
}
public void Disconnect()
{
if (_sftpClient != null)
{
_sftpClient.Disconnect();
}
if (_scp != null)
{
_scp.Close();
}
}
public static bool UploadScp(string server, string localFile, string remoteFile, string username, string password)
{
try {
SshTransferProtocolBase sshCp = new Scp(server, username, password);
sshCp.Connect();
sshCp.Put(localFile, remoteFile);
sshCp.Close();
return true;
} catch (Exception e) {
nlogger.ErrorException("while uploading this happened", e);
return false;
}
}
private static void ScpPut(string host, string from, string to)
{
var scp = new SSH.Scp(host.Split(':')[0], User, Password);
if (host.Contains(":"))
{
scp.Connect(Int32.Parse(host.Split(':')[1]));
}
else
{
scp.Connect();
}
scp.Recursive = true;
scp.Put(from, to);
scp.Close();
}
public static void TransferFileToMachine(string localFilePath, string remoteFilePath)
{
try
{
//Create a new SCP instance
Scp scp = new Scp(MACHINE_IP, USER_NAME, PASSWORD);
scp.Connect();
//Copy a file from remote SSH server to local machine
scp.To(localFilePath, remoteFilePath);
scp.Close();
}
catch (Exception)
{
throw;
}
}
private static void ScpPut(string host, string from, string to)
{
var scp = new SSH.Scp(host.Split(':')[0], User, Password);
if (host.Contains(":"))
{
scp.Connect(Int32.Parse(host.Split(':')[1]));
}
else
{
scp.Connect();
}
scp.Recursive = true;
scp.Put(from, to);
scp.Close();
}
public object Run(TaskRunContext task)
{
lock (_gridLock)
{
RefreshCertificate();
//var incarnation = task.Incarnation;
string tmpFileName = null;
if (task.UserCert != null)
{
Log.Info("Using user's certificate");
tmpFileName = Path.GetTempFileName();
IOProxy.Storage.Download(task.UserCert, tmpFileName);
var scpForCert = new SSH.Scp(HELPER_SSH_HOST, HELPER_SSH_USER, HELPER_SSH_PASS);
scpForCert.Connect();
scpForCert.Recursive = true;
scpForCert.Put(tmpFileName, "/tmp/x509up_u500");
scpForCert.Close();
File.Delete(tmpFileName);
SshExec(PilotCommands.SetPermissionsOnProxyCertFile);
}
else
{
Log.Info("Using system's certificate");
}
try
{
long coresToUse = task.NodesConfig.Sum(cfg => cfg.Cores);
var node = GetNode(task);
var pack = node.PackageByName(task.PackageName);
// todo : remove
string commandLine = task.CommandLine;
commandLine = commandLine.Replace("java -jar ", "");
if (task.PackageName.ToLowerInvariant() == "cnm")
{
commandLine = commandLine.Replace("{0}", "ru.ifmo.hpc.main.ExtendedModel");
}
else
if (task.PackageName.ToLowerInvariant() == "ism")
{
commandLine = commandLine.Replace("{0}", "ru.ifmo.hpc.main.SpreadModel");
}
else
{
//if (task.PackageName.ToLowerInvariant() == "orca")
commandLine = commandLine.Replace("{0}", "");
}
string ftpFolderFromSystem = IncarnationParams.IncarnatePath(node.DataFolders.ExchangeUrlFromSystem, task.TaskId, CopyPhase.In);
string ftpFolderFromResource = IncarnationParams.IncarnatePath(node.DataFolders.ExchangeUrlFromResource, task.TaskId, CopyPhase.In);
string gridFtpFolder = IncarnationParams.IncarnatePath(node.DataFolders.LocalFolder, task.TaskId, CopyPhase.None);
SshExec(PilotCommands.MakeFolderOnGridFtp, gridFtpFolder);
string endl = "\n";
// Сначала дописываем недостающий входной файл (скрипт запуска пакета на кластере)
string scriptName = pack.AppPath;
//if (pack.EnvVars.Any())
{
// Файл с установкой переменных окружения, если пакет их использует
scriptName = "run.sh";
var scriptContent = new StringBuilder();
scriptContent.Append("#!/bin/bash" + endl);
foreach (var pair in pack.EnvVars)
{
scriptContent.AppendFormat("export {0}={1}" + endl, pair.Key, pair.Value);
}
scriptContent.Append(pack.AppPath);
/*
* if (task.PackageName.ToLowerInvariant() == "orca")
* {
* string[] args = commandLine.Split(new[] { ' ' }, StringSplitOptions.RemoveEmptyEntries);
* for (int i = 0; i < args.Length; i++)
* {
* if (args[i] == "orca.out")
* scriptContent.Append(" >");
*
* scriptContent.Append(" $" + (i + 1).ToString());
* }
* }
* else*/
{
scriptContent.Append(" " + commandLine);
}
string scriptLocalPath = Path.GetTempFileName();
File.WriteAllText(scriptLocalPath, scriptContent.ToString());
IOProxy.Ftp.UploadLocalFile(scriptLocalPath, ftpFolderFromSystem, scriptName);
File.Delete(scriptLocalPath);
}
//IOProxy.Ftp.UploadLocalFile(DEFAULT_JOB_LAUNCHER_PATH, GetFtpInputFolder(taskId), Path.GetFileName(DEFAULT_JOB_LAUNCHER_PATH));
// Копируем входные файлы с ФТП на ГридФТП
SshExec(PilotCommands.CopyFilesToGridFtp, ftpFolderFromResource + " " + gridFtpFolder);
SshExec(PilotCommands.MakeFilesExecutableOnGridFtp, gridFtpFolder + "*");
// Формируем описание задания для грида
var jobFileContent = new StringBuilder();
jobFileContent.AppendFormat(@"{{ ""version"": 2, ""description"": ""{0}""," + endl, task.TaskId);
jobFileContent.AppendFormat(@" ""default_storage_base"": ""{0}""," + endl, gridFtpFolder);
jobFileContent.AppendFormat(@" ""tasks"": [ {{ ""id"": ""a"", ""description"": ""task"", ""definition"": {{ ""version"": 2," + endl);
jobFileContent.AppendFormat(@" ""executable"": ""{0}""," + endl, scriptName);
//jobFileContent.AppendFormat(@" ""arguments"": [ ""{0}"" ]," + endl, String.Join(@""", """, args));
jobFileContent.AppendFormat(@" ""input_files"": {{" + endl);
if (scriptName == "run.sh") // todo : if no input files?
{
jobFileContent.AppendFormat(@" ""run.sh"": ""run.sh""," + endl);
}
jobFileContent.AppendFormat(@" " + String.Join(
"," + endl + " ",
task.InputFiles.Select(
file => String.Format(@"""{0}"": ""{0}""", file.FileName)
)
));
jobFileContent.AppendFormat(endl + @" }}," + endl);
jobFileContent.AppendFormat(@" ""output_files"": {{" + endl);
//if (task.PackageName.ToLowerInvariant() == "cnm")
// jobFileContent.AppendFormat(@" ""output.dat"": ""output.dat""" + endl);
//else
if (task.PackageName.ToLowerInvariant() == "ism")
{
jobFileContent.AppendFormat(@" ""output.dat"": ""output.dat""" + endl);
}
else
if (task.PackageName.ToLowerInvariant() == "orca")
{
jobFileContent.AppendFormat(@" ""orca.out"": ""orca.out""," + endl);
jobFileContent.AppendFormat(@" ""eldens.cube"": ""eldens.cube""" + endl);
}
else
{
jobFileContent.AppendFormat(@" " + String.Join(
"," + endl + " ",
task.ExpectedOutputFileNames
.Where(name => name != "std.out" && name != "std.err")
.Select(
name => String.Format(@"""{0}"": ""{0}""", name)
)
) + endl);
}
jobFileContent.AppendFormat(@" }}," + endl);
jobFileContent.AppendFormat(@" ""stdout"": ""std.out"", ""stderr"": ""std.err"", " + endl);
jobFileContent.AppendFormat(@" ""count"": {0}" + endl, coresToUse);
if (pack.Params.ContainsKey("requirements"))
{
jobFileContent.AppendFormat(@" ,""requirements"": {0}" + endl, pack.Params["requirements"]);
}
jobFileContent.AppendFormat(@" }} }} ]," + endl);
jobFileContent.AppendFormat(@" ""requirements"": {{ ""hostname"": [""{0}""]", node.NodeAddress);
//if (pack.Params.ContainsKey("requirements"))
// jobFileContent.AppendFormat(@", {0}" + endl, pack.Params["requirements"]);
jobFileContent.AppendFormat(@"}}" + endl + "}}", node.NodeAddress);
Log.Debug(String.Format("Task's '{0}' grid job JSON: ", task.TaskId, jobFileContent));
string jobFileName = "job_" + task.TaskId.ToString() + ".js";
string jobFilePathOnHelper = JOBS_FOLDER_ON_HELPER + jobFileName;
//string jobFileContent = File.ReadAllText(DEFAULT_JOB_DESCR_PATH).Replace(GRIDFTP_PATH_TOKEN, taskFolderOnGridFtp);
string jobFilePathLocal = Path.GetTempFileName();
File.WriteAllText(jobFilePathLocal, jobFileContent.ToString());
// Записываем его на сервер с Пилотом
var scp = new SSH.Scp(HELPER_SSH_HOST, HELPER_SSH_USER, HELPER_SSH_PASS);
/*
* var notifier = new JobDescriptionUploadNotifier(TaskId, Cluster, RunParams);
* scp.OnTransferEnd += new SSH.FileTransferEvent(notifier.OnFinish); // todo : необязательно
*/
scp.Connect();
scp.Recursive = true;
scp.Put(jobFilePathLocal, jobFilePathOnHelper);
scp.Close();
File.Delete(jobFilePathLocal); // todo : remove files on helper and gridftp
// Запускаем
Log.Info(String.Format(
"Trying to exec task {0} on grid cluster {1}",
task.TaskId, node.NodeName
));
string launchResult = SshExec(PilotCommands.SubmitJob, jobFilePathOnHelper, pilotUrl: node.Services.ExecutionUrl);
int urlPos = launchResult.IndexOf("https://");
string jobUrl = launchResult.Substring(urlPos).Trim() + "a";
Log.Debug(jobUrl);
Log.Info(String.Format(
"Task {0} launched on grid with jobUrl = {1}",
task.TaskId, jobUrl
));
return(jobUrl);
}
catch (Exception e)
{
Log.Error(String.Format(
"Error while starting task {0} in grid: {1}\n{2}",
task.TaskId, e.Message, e.StackTrace
));
throw;
}
finally
{
if (task.UserCert != null)
{
Log.Info("Wiping user's certificate");
tmpFileName = Path.GetTempFileName();
File.WriteAllText(tmpFileName, "Wiped by Easis system");
var scpForCert = new SSH.Scp(HELPER_SSH_HOST, HELPER_SSH_USER, HELPER_SSH_PASS);
scpForCert.Connect();
scpForCert.Recursive = true;
scpForCert.Put(tmpFileName, "/tmp/x509up_u500");
scpForCert.Close();
File.Delete(tmpFileName);
SshExec(PilotCommands.SetPermissionsOnProxyCertFile);
}
}
}
}
public object Run(TaskRunContext task)
{
lock (_gridLock)
{
RefreshCertificate();
//var incarnation = task.Incarnation;
string tmpFileName = null;
if (task.UserCert != null)
{
Log.Info("Using user's certificate");
tmpFileName = Path.GetTempFileName();
IOProxy.Storage.Download(task.UserCert, tmpFileName);
var scpForCert = new SSH.Scp(HELPER_SSH_HOST, HELPER_SSH_USER, HELPER_SSH_PASS);
scpForCert.Connect();
scpForCert.Recursive = true;
scpForCert.Put(tmpFileName, "/tmp/x509up_u500");
scpForCert.Close();
File.Delete(tmpFileName);
SshExec(PilotCommands.SetPermissionsOnProxyCertFile);
}
else
{
Log.Info("Using system's certificate");
}
try
{
long coresToUse = task.NodesConfig.Sum(cfg => cfg.Cores);
var node = GetNode(task);
var pack = node.PackageByName(task.PackageName);
// todo : remove
string commandLine = task.CommandLine;
commandLine = commandLine.Replace("java -jar ", "");
if (task.PackageName.ToLowerInvariant() == "cnm")
commandLine = commandLine.Replace("{0}", "ru.ifmo.hpc.main.ExtendedModel");
else
if (task.PackageName.ToLowerInvariant() == "ism")
commandLine = commandLine.Replace("{0}", "ru.ifmo.hpc.main.SpreadModel");
else
//if (task.PackageName.ToLowerInvariant() == "orca")
commandLine = commandLine.Replace("{0}", "");
string ftpFolderFromSystem = IncarnationParams.IncarnatePath(node.DataFolders.ExchangeUrlFromSystem, task.TaskId, CopyPhase.In);
string ftpFolderFromResource = IncarnationParams.IncarnatePath(node.DataFolders.ExchangeUrlFromResource, task.TaskId, CopyPhase.In);
string gridFtpFolder = IncarnationParams.IncarnatePath(node.DataFolders.LocalFolder, task.TaskId, CopyPhase.None);
SshExec(PilotCommands.MakeFolderOnGridFtp, gridFtpFolder);
string endl = "\n";
// Сначала дописываем недостающий входной файл (скрипт запуска пакета на кластере)
string scriptName = pack.AppPath;
//if (pack.EnvVars.Any())
{
// Файл с установкой переменных окружения, если пакет их использует
scriptName = "run.sh";
var scriptContent = new StringBuilder();
scriptContent.Append("#!/bin/bash" + endl);
foreach (var pair in pack.EnvVars)
scriptContent.AppendFormat("export {0}={1}" + endl, pair.Key, pair.Value);
scriptContent.Append(pack.AppPath);
/*
if (task.PackageName.ToLowerInvariant() == "orca")
{
string[] args = commandLine.Split(new[] { ' ' }, StringSplitOptions.RemoveEmptyEntries);
for (int i = 0; i < args.Length; i++)
{
if (args[i] == "orca.out")
scriptContent.Append(" >");
scriptContent.Append(" $" + (i + 1).ToString());
}
}
else*/
{
scriptContent.Append(" " + commandLine);
}
string scriptLocalPath = Path.GetTempFileName();
File.WriteAllText(scriptLocalPath, scriptContent.ToString());
IOProxy.Ftp.UploadLocalFile(scriptLocalPath, ftpFolderFromSystem, scriptName);
File.Delete(scriptLocalPath);
}
//IOProxy.Ftp.UploadLocalFile(DEFAULT_JOB_LAUNCHER_PATH, GetFtpInputFolder(taskId), Path.GetFileName(DEFAULT_JOB_LAUNCHER_PATH));
// Копируем входные файлы с ФТП на ГридФТП
SshExec(PilotCommands.CopyFilesToGridFtp, ftpFolderFromResource + " " + gridFtpFolder);
SshExec(PilotCommands.MakeFilesExecutableOnGridFtp, gridFtpFolder + "*");
// Формируем описание задания для грида
var jobFileContent = new StringBuilder();
jobFileContent.AppendFormat(@"{{ ""version"": 2, ""description"": ""{0}""," + endl, task.TaskId);
jobFileContent.AppendFormat(@" ""default_storage_base"": ""{0}""," + endl, gridFtpFolder);
jobFileContent.AppendFormat(@" ""tasks"": [ {{ ""id"": ""a"", ""description"": ""task"", ""definition"": {{ ""version"": 2," + endl);
jobFileContent.AppendFormat(@" ""executable"": ""{0}""," + endl, scriptName);
//jobFileContent.AppendFormat(@" ""arguments"": [ ""{0}"" ]," + endl, String.Join(@""", """, args));
jobFileContent.AppendFormat(@" ""input_files"": {{" + endl);
if (scriptName == "run.sh") // todo : if no input files?
jobFileContent.AppendFormat(@" ""run.sh"": ""run.sh""," + endl);
jobFileContent.AppendFormat(@" " + String.Join(
"," + endl + " ",
task.InputFiles.Select(
file => String.Format(@"""{0}"": ""{0}""", file.FileName)
)
));
jobFileContent.AppendFormat(endl + @" }}," + endl);
jobFileContent.AppendFormat(@" ""output_files"": {{" + endl);
//if (task.PackageName.ToLowerInvariant() == "cnm")
// jobFileContent.AppendFormat(@" ""output.dat"": ""output.dat""" + endl);
//else
if (task.PackageName.ToLowerInvariant() == "ism")
jobFileContent.AppendFormat(@" ""output.dat"": ""output.dat""" + endl);
else
if (task.PackageName.ToLowerInvariant() == "orca")
{
jobFileContent.AppendFormat(@" ""orca.out"": ""orca.out""," + endl);
jobFileContent.AppendFormat(@" ""eldens.cube"": ""eldens.cube""" + endl);
}
else
{
jobFileContent.AppendFormat(@" " + String.Join(
"," + endl + " ",
task.ExpectedOutputFileNames
.Where(name => name != "std.out" && name != "std.err")
.Select(
name => String.Format(@"""{0}"": ""{0}""", name)
)
) + endl);
}
jobFileContent.AppendFormat(@" }}," + endl);
jobFileContent.AppendFormat(@" ""stdout"": ""std.out"", ""stderr"": ""std.err"", " + endl);
jobFileContent.AppendFormat(@" ""count"": {0}" + endl, coresToUse);
if (pack.Params.ContainsKey("requirements"))
jobFileContent.AppendFormat(@" ,""requirements"": {0}" + endl, pack.Params["requirements"]);
jobFileContent.AppendFormat(@" }} }} ]," + endl);
jobFileContent.AppendFormat(@" ""requirements"": {{ ""hostname"": [""{0}""]", node.NodeAddress);
//if (pack.Params.ContainsKey("requirements"))
// jobFileContent.AppendFormat(@", {0}" + endl, pack.Params["requirements"]);
jobFileContent.AppendFormat(@"}}" + endl + "}}", node.NodeAddress);
Log.Debug(String.Format("Task's '{0}' grid job JSON: ", task.TaskId, jobFileContent));
string jobFileName = "job_" + task.TaskId.ToString() + ".js";
string jobFilePathOnHelper = JOBS_FOLDER_ON_HELPER + jobFileName;
//string jobFileContent = File.ReadAllText(DEFAULT_JOB_DESCR_PATH).Replace(GRIDFTP_PATH_TOKEN, taskFolderOnGridFtp);
string jobFilePathLocal = Path.GetTempFileName();
File.WriteAllText(jobFilePathLocal, jobFileContent.ToString());
// Записываем его на сервер с Пилотом
var scp = new SSH.Scp(HELPER_SSH_HOST, HELPER_SSH_USER, HELPER_SSH_PASS);
/*
var notifier = new JobDescriptionUploadNotifier(TaskId, Cluster, RunParams);
scp.OnTransferEnd += new SSH.FileTransferEvent(notifier.OnFinish); // todo : необязательно
*/
scp.Connect();
scp.Recursive = true;
scp.Put(jobFilePathLocal, jobFilePathOnHelper);
scp.Close();
File.Delete(jobFilePathLocal); // todo : remove files on helper and gridftp
// Запускаем
Log.Info(String.Format(
"Trying to exec task {0} on grid cluster {1}",
task.TaskId, node.NodeName
));
string launchResult = SshExec(PilotCommands.SubmitJob, jobFilePathOnHelper, pilotUrl: node.Services.ExecutionUrl);
int urlPos = launchResult.IndexOf("https://");
string jobUrl = launchResult.Substring(urlPos).Trim() + "a";
Log.Debug(jobUrl);
Log.Info(String.Format(
"Task {0} launched on grid with jobUrl = {1}",
task.TaskId, jobUrl
));
return jobUrl;
}
catch (Exception e)
{
Log.Error(String.Format(
"Error while starting task {0} in grid: {1}\n{2}",
task.TaskId, e.Message, e.StackTrace
));
throw;
}
finally
{
if (task.UserCert != null)
{
Log.Info("Wiping user's certificate");
tmpFileName = Path.GetTempFileName();
File.WriteAllText(tmpFileName, "Wiped by Easis system");
var scpForCert = new SSH.Scp(HELPER_SSH_HOST, HELPER_SSH_USER, HELPER_SSH_PASS);
scpForCert.Connect();
scpForCert.Recursive = true;
scpForCert.Put(tmpFileName, "/tmp/x509up_u500");
scpForCert.Close();
File.Delete(tmpFileName);
SshExec(PilotCommands.SetPermissionsOnProxyCertFile);
}
}
}
}