Exemplo n.º 1
0
        public Task(TaskDescription description)
            : base(description)
        {
            if (Priority == TaskPriority.Urgent)
            {
                if (!ExecParams.ContainsKey("MinTime"))
                    ExecParams["MinTime"] = "0";

                if (!ExecParams.ContainsKey("MaxTime"))
                    ExecParams["MaxTime"] = ExecParams["MinTime"];
            }

            OutputParams = new Dictionary<string, string>();

            Time = new TaskTimeMeasurement();
            Time.Started(TaskTimeMetric.Postponed);
            if (this.IsFake())
                Time.Edge(started: TaskTimeMetric.Queued, finished: TaskTimeMetric.Postponed);

            Estimations = null;
            CurrentSchedule = null;
            Incarnation = new IncarnationParams();

            State = TaskState.Defined;

            string stepName = ExecParams.ContainsKey("StepName")? ExecParams["StepName"]: null;
            string storageRoot = IOProxy.Storage.BuildPath(UserId, WfId, stepId: TaskId.ToString(), stepName: stepName);
            this.PackageEngineState = new PackageEngineState(description, storageRoot); // todo : measure PackageEngineState expenses
        }
Exemplo n.º 2
0
        public Task(Task otherTask)
            : base(otherTask)
        {
            if (otherTask.OutputParams != null)
                OutputParams = new Dictionary<string, string>(otherTask.OutputParams);

            if (otherTask.PackageEngineState != null)
                this.PackageEngineState = (PackageEngineState) otherTask.PackageEngineState.Clone();

            Time = new TaskTimeMeasurement(otherTask.Time);
            Incarnation = new IncarnationParams(otherTask.Incarnation);

            if (otherTask.Estimations != null)
                Estimations = new Dictionary<string, double>(otherTask.Estimations);

            CurrentSchedule = null;
            if (otherTask.CurrentSchedule != null)
            {
                CurrentSchedule = new TaskSchedule(otherTask.CurrentSchedule);

                // immutable:
                this.AssignedResource = otherTask.AssignedResource;
                this.AssignedNodes = otherTask.AssignedNodes;
            }

            State = otherTask.State;

            _inputsProcessed = otherTask._inputsProcessed;
            _inputsProcessingError = otherTask._inputsProcessingError;

            _failReason = otherTask._failReason;
            if (otherTask._lastEvent != null && otherTask._lastEvent.HasValue)
                _lastEvent = otherTask._lastEvent.Value;
        }
Exemplo n.º 3
0
        private string GetFtpFolder(NodeConfig nodeConfig, Resource resource, CopyPhase phase)
        {
            string ftpFolder           = resource.Nodes.First(n => n.NodeName == nodeConfig.NodeName).DataFolders.ExchangeUrlFromSystem;
            string incarnatedFtpFolder = IncarnationParams.IncarnatePath(ftpFolder, TaskId, phase);

            return(incarnatedFtpFolder);
        }
Exemplo n.º 4
0
        public IncarnationParams(IncarnationParams otherIncarnation)
            : this()
        {
            if (otherIncarnation != null)
            {
                this.CommandLine = otherIncarnation.CommandLine;
                //this.StdInFile   = otherIncarnation.StdInFile;
                //this.StdOutFile  = otherIncarnation.StdOutFile;

                //this.PackageNameInConfig = otherIncarnation.PackageNameInConfig;

                this.PackageName = otherIncarnation.PackageName;
                //this.ProvidedTaskId = otherIncarnation.ProvidedTaskId;

                if (otherIncarnation.FilesToCopy.Any())
                {
                    FilesToCopy = otherIncarnation.FilesToCopy.Select(descr => descr).ToArray(); // todo : maybe Incarnation.FilesToCopy IEnumerable -> array[]?
                }
                if (otherIncarnation.ExpectedOutputFileNames.Any())
                {
                    ExpectedOutputFileNames = otherIncarnation.ExpectedOutputFileNames.Select(name => name).ToArray(); // todo : maybe Incarnation.ExpectedOutputFileNames IEnumerable -> array[]?
                }
                CanExpectMoreFiles = otherIncarnation.CanExpectMoreFiles;

                //this.InputFolderPath = otherIncarnation.InputFolderPath;
                //this.OutputFolderPath = otherIncarnation.OutputFolderPath;
            }
        }
Exemplo n.º 5
0
        public IncarnationParams(IncarnationParams otherIncarnation)
            : this()
        {
            if (otherIncarnation != null)
            {
                this.CommandLine = otherIncarnation.CommandLine;
                //this.StdInFile   = otherIncarnation.StdInFile;
                //this.StdOutFile  = otherIncarnation.StdOutFile;

                //this.PackageNameInConfig = otherIncarnation.PackageNameInConfig;

                this.PackageName = otherIncarnation.PackageName;
                //this.ProvidedTaskId = otherIncarnation.ProvidedTaskId;

                if (otherIncarnation.FilesToCopy.Any())
                    FilesToCopy = otherIncarnation.FilesToCopy.Select(descr => descr).ToArray(); // todo : maybe Incarnation.FilesToCopy IEnumerable -> array[]?

                if (otherIncarnation.ExpectedOutputFileNames.Any())
                    ExpectedOutputFileNames = otherIncarnation.ExpectedOutputFileNames.Select(name => name).ToArray(); // todo : maybe Incarnation.ExpectedOutputFileNames IEnumerable -> array[]?
                CanExpectMoreFiles = otherIncarnation.CanExpectMoreFiles;

                //this.InputFolderPath = otherIncarnation.InputFolderPath;
                //this.OutputFolderPath = otherIncarnation.OutputFolderPath;
            }
        }
Exemplo n.º 6
0
        public static IncarnationParams ProcessInputFiles(/* ref */ PackageEngineState engineState, out TimeSpan inputFilesTime)
        {
            var engine = new PackageEngine(engineState.CompiledDef, engineState.EngineContext);

            var incarnation = new IncarnationParams();
            var fileManager = new InputFileManager(engineState.StoragePathBase);

            inputFilesTime = TimeSpanExt.Measure(() =>
            {
                TimeSpan timeLimit = PB_PROCESS_TIME_LIMIT;

                try
                {
                    Tools.ProcessWithTimeLimit(timeLimit, () =>
                    {
                        engine.PrepareInputs(fileManager);
                        CheckEngineResult(engine);
                    });
                }
                catch (TimeLimitException tle)
                {
                    throw new PackageBaseException("Inputs processing in Package base exceeded time limit (" + timeLimit.TotalSeconds + " seconds).", tle);
                }
                finally
                {
                    incarnation.FilesToCopy = incarnation.FilesToCopy.Concat(fileManager.GetInputFiles()).ToArray();
                    fileManager.Cleanup();
                }
            });

            object cmdLine = engine.Ctx.Memory[CmdLineDef.DefName].Value;

            //if (cmdLine == null)
            //throw new PackageBaseException("CommandLine generated by PackageBase is empty");
            incarnation.CommandLine = cmdLine.ToString();
            Log.Debug(String.Format(
                          "Command line for pack {0}.{1} is '{2}' (task {3}). Cmdline object from PB is {4}",
                          engine.CompiledMode.ModeQName.PackageName ?? "''", engine.CompiledMode.ModeQName.ModeName ?? "''",
                          incarnation.CommandLine ?? "", engineState._taskDescription.TaskId,
                          cmdLine == null ? "null" : "not null"
                          ));

            if (String.IsNullOrWhiteSpace(incarnation.CommandLine))
            {
                throw new PackageBaseException("CommandLine generated by PackageBase is empty");
            }

            if (!engine.IsReadyToRun())
            {
                Log.Warn("PackageBase: task is not ready to run after PrepareInputs");

                Log.Error("PackageBase: task is not ready to run after PrepareInputs");
                throw new PackageBaseException("PackageBase: task is not ready to run after processing inputs");
            }

            return(incarnation);
        }
Exemplo n.º 7
0
        public static IncarnationParams ProcessInputFiles(/* ref */ PackageEngineState engineState, out TimeSpan inputFilesTime)
        {
            var engine = new PackageEngine(engineState.CompiledDef, engineState.EngineContext);

            var incarnation = new IncarnationParams();
            var fileManager = new InputFileManager(engineState.StoragePathBase);

            inputFilesTime = TimeSpanExt.Measure(() =>
            {
                TimeSpan timeLimit = PB_PROCESS_TIME_LIMIT;

                try
                {
                    Tools.ProcessWithTimeLimit(timeLimit, () =>
                    {
                        engine.PrepareInputs(fileManager);
                        CheckEngineResult(engine);
                    });
                }
                catch (TimeLimitException tle)
                {
                    throw new PackageBaseException("Inputs processing in Package base exceeded time limit (" + timeLimit.TotalSeconds + " seconds).", tle);
                }
                finally
                {
                    incarnation.FilesToCopy = incarnation.FilesToCopy.Concat(fileManager.GetInputFiles()).ToArray();
                    fileManager.Cleanup();
                }
            });

            object cmdLine = engine.Ctx.Memory[CmdLineDef.DefName].Value;
            //if (cmdLine == null)
                //throw new PackageBaseException("CommandLine generated by PackageBase is empty");
            incarnation.CommandLine = cmdLine.ToString();
            Log.Debug(String.Format(
                "Command line for pack {0}.{1} is '{2}' (task {3}). Cmdline object from PB is {4}",
                engine.CompiledMode.ModeQName.PackageName ?? "''", engine.CompiledMode.ModeQName.ModeName ?? "''",
                incarnation.CommandLine ?? "", engineState._taskDescription.TaskId,
                cmdLine == null ? "null" : "not null"
            ));

            if (String.IsNullOrWhiteSpace(incarnation.CommandLine))
                throw new PackageBaseException("CommandLine generated by PackageBase is empty");

            if (!engine.IsReadyToRun())
            {
                Log.Warn("PackageBase: task is not ready to run after PrepareInputs");

                Log.Error("PackageBase: task is not ready to run after PrepareInputs");
                throw new PackageBaseException("PackageBase: task is not ready to run after processing inputs");
            }

            return incarnation;
        }
Exemplo n.º 8
0
        public override string Run(ulong taskId, IncarnationParams incarnation, Resource resource, IEnumerable <NodeConfig> nodesConfig)
        {
            lock (_lock)
            {
                int coresToUse = nodesConfig.Sum(conf => conf.Cores);
                var node       = GetDefaultNodeSettings(resource, nodesConfig);
                var pack       = node.PackageByName(incarnation.PackageName);

                //string ftpFolder = IncarnationParams.IncarnatePath(node.DataFolders.ExchangeUrlFromSystem, taskId, CopyPhase.In);
                //string jobFtpFolder = IncarnationParams.IncarnatePath(node.DataFolders.ExchangeUrlFromSystem, taskId, CopyPhase.None);
                string ftpInputFolder    = IncarnationParams.IncarnatePath(node.DataFolders.ExchangeUrlFromResource, taskId, CopyPhase.In);
                string ftpOutputFolder   = IncarnationParams.IncarnatePath(node.DataFolders.ExchangeUrlFromResource, taskId, CopyPhase.Out);
                string clusterHomeFolder = IncarnationParams.IncarnatePath(node.DataFolders.LocalFolder, taskId, CopyPhase.None);

                //IOProxy.Ftp.MakePath(ftpInputFolder);
                //IOProxy.Ftp.MakePath(ftpOutputFolder);

                try
                {
                    SshExec(node, "mkdir " + clusterHomeFolder);
                }
                catch (Exception e)
                {
                    Log.Warn(e.Message);
                }

                string fileNames = ""; //String.Join(" ", incarnation.FilesToCopy.Select(f => f.FileName));
                foreach (var file in incarnation.FilesToCopy)
                {
                    string tmpFile = System.IO.Path.GetTempFileName();
                    IOProxy.Storage.Download(file.StorageId, tmpFile);

                    string fileOnCluster = clusterHomeFolder.TrimEnd(new[] { '/', '\\' }) + "/" + file.FileName;
                    fileNames += " " + fileOnCluster;

                    ScpCopy(node, fileOnCluster, tmpFile);
                    System.IO.File.Delete(tmpFile);
                }

                string cmdLine = String.Format(incarnation.CommandLine, pack.AppPath, "clavire" + taskId.ToString(), fileNames.Trim());

                string result = SshExec(node, cmdLine);
                string jobId  = result.Split(new[] { '\r', '\n' }, StringSplitOptions.RemoveEmptyEntries).First();

                _nodeUsed[node.NodeName] = true;

                return(jobId);
            }
        }
Exemplo n.º 9
0
        public Task(Task otherTask)
            : base(otherTask)
        {
            if (otherTask.OutputParams != null)
            {
                OutputParams = new Dictionary <string, string>(otherTask.OutputParams);
            }

            if (otherTask.PackageEngineState != null)
            {
                this.PackageEngineState = (PackageEngineState)otherTask.PackageEngineState.Clone();
            }

            Time        = new TaskTimeMeasurement(otherTask.Time);
            Incarnation = new IncarnationParams(otherTask.Incarnation);

            if (otherTask.Estimations != null)
            {
                Estimations = new Dictionary <string, double>(otherTask.Estimations);
            }

            CurrentSchedule = null;
            if (otherTask.CurrentSchedule != null)
            {
                CurrentSchedule = new TaskSchedule(otherTask.CurrentSchedule);

                // immutable:
                this.AssignedResource = otherTask.AssignedResource;
                this.AssignedNodes    = otherTask.AssignedNodes;
            }

            State = otherTask.State;

            _inputsProcessed       = otherTask._inputsProcessed;
            _inputsProcessingError = otherTask._inputsProcessingError;

            _failReason = otherTask._failReason;
            if (otherTask._lastEvent != null && otherTask._lastEvent.HasValue)
            {
                _lastEvent = otherTask._lastEvent.Value;
            }
        }
Exemplo n.º 10
0
        public override Tuple <TaskState, string> GetTaskState(ulong taskId, string providedTaskId, Resource resource, IEnumerable <NodeConfig> nodesConfig)
        {
            lock (_gridLock)
            {
                RefreshCertificate();

                string taskIdString = taskId.ToString();
                string state        = SshExec(PilotCommands.JobStatus, providedTaskId).ToLower();

                if (state.Contains("is new"))
                {
                    return(Tuple.Create(TaskState.Started, state));
                }
                //return Tuple.Create(TaskState.Scheduled, state);

                if (state.Contains("is running") || state.Contains("is starting"))
                {
                    return(Tuple.Create(TaskState.Started, state));
                }

                var    node = GetDefaultNodeSettings(resource, nodesConfig);
                string ftpOutFolderFromSystem   = IncarnationParams.IncarnatePath(node.DataFolders.ExchangeUrlFromSystem, taskId, CopyPhase.Out);
                string ftpOutFolderFromResource = IncarnationParams.IncarnatePath(node.DataFolders.ExchangeUrlFromResource, taskId, CopyPhase.Out);
                string gridFolder = IncarnationParams.IncarnatePath(node.DataFolders.LocalFolder, taskId, CopyPhase.None);

                IOProxy.Ftp.MakePath(ftpOutFolderFromSystem);
                SshExec(PilotCommands.CopyFilesToGridFtp, gridFolder + " " + ftpOutFolderFromResource);

                _nodeUsed[node.NodeName] = false;

                if (state.Contains("is finished"))
                {
                    return(Tuple.Create(TaskState.Completed, state));
                }
                else
                {
                    return(Tuple.Create(TaskState.Failed, state));
                }
            }
        }
Exemplo n.º 11
0
        public override Tuple <TaskState, string> GetTaskState(ulong taskId, string providedTaskId, Resource resource, IEnumerable <NodeConfig> nodesConfig)
        {
            lock (_lock)
            {
                string result = SshExec(resource.Nodes.First(), SshCommands.GetTaskState, providedTaskId).ToLowerInvariant();

                if (result.Contains("job_state = R") || result.Contains("job_state = Q"))
                {
                    return(new Tuple <TaskState, string>(TaskState.Started, result));
                }
                // esle if (Aborted, Failed)
                else
                {
                    var    node = GetDefaultNodeSettings(resource, nodesConfig);
                    string ftpOutFolderFromSystem = IncarnationParams.IncarnatePath(node.DataFolders.ExchangeUrlFromSystem, taskId, CopyPhase.Out);
                    string clusterFolder          = "share/ANSYS_JOBS/clavire" + taskId.ToString() + "_0001/";

                    IOProxy.Ftp.MakePath(ftpOutFolderFromSystem);

                    var fileNames =
                        SshExec(node, SshCommands.Ls, clusterFolder)
                        .Split(new[] { ", " }, StringSplitOptions.RemoveEmptyEntries)
                        .Where(st => !st.Contains("/"))
                        .Select(st => st.Replace("*", "").Replace("|", ""))
                        .Where(st => !st.Contains(".rst") && !st.Contains(".err") && !st.Contains(".esav"));

                    foreach (string fileName in fileNames)
                    {
                        string tmpFile = System.IO.Path.GetTempFileName();
                        ScpGet(node, clusterFolder + fileName, tmpFile);
                        IOProxy.Ftp.UploadLocalFile(tmpFile, ftpOutFolderFromSystem, fileName);
                        System.IO.File.Delete(tmpFile);
                    }

                    _nodeUsed[node.NodeName] = false;

                    return(new Tuple <TaskState, string>(TaskState.Completed, result));
                }
            }
        }
Exemplo n.º 12
0
        public Task(TaskDescription description)
            : base(description)
        {
            if (Priority == TaskPriority.Urgent)
            {
                if (!ExecParams.ContainsKey("MinTime"))
                {
                    ExecParams["MinTime"] = "0";
                }

                if (!ExecParams.ContainsKey("MaxTime"))
                {
                    ExecParams["MaxTime"] = ExecParams["MinTime"];
                }
            }

            OutputParams = new Dictionary <string, string>();

            Time = new TaskTimeMeasurement();
            Time.Started(TaskTimeMetric.Postponed);
            if (this.IsFake())
            {
                Time.Edge(started: TaskTimeMetric.Queued, finished: TaskTimeMetric.Postponed);
            }

            Estimations     = null;
            CurrentSchedule = null;
            Incarnation     = new IncarnationParams();

            State = TaskState.Defined;

            string stepName    = ExecParams.ContainsKey("StepName")? ExecParams["StepName"]: null;
            string storageRoot = IOProxy.Storage.BuildPath(UserId, WfId, stepId: TaskId.ToString(), stepName: stepName);

            this.PackageEngineState = new PackageEngineState(description, storageRoot); // todo : measure PackageEngineState expenses
        }
Exemplo n.º 13
0
        public override string Run(ulong taskId, IncarnationParams incarnation, Resource resource, IEnumerable<NodeConfig> nodesConfig)
        {
            lock (_pcLock)
            {
                //AcceptPsToolsEula();

                int coresToUse = nodesConfig.Sum(conf => conf.Cores);
                var node = GetDefaultNodeSettings(resource, nodesConfig);

                if (_nodeUsed[node.NodeName])
                    throw new Exception(String.Format("Could not run task {0} on node {1}: node used by another task", taskId, node.NodeName));

                string ftpFolder = IncarnationParams.IncarnatePath(node.DataFolders.ExchangeUrlFromSystem, taskId, CopyPhase.In);
                string jobFtpFolder = IncarnationParams.IncarnatePath(node.DataFolders.ExchangeUrlFromSystem, taskId, CopyPhase.None);
                string sharedInputFolder  = IncarnationParams.IncarnatePath(node.DataFolders.ExchangeUrlFromResource, taskId, CopyPhase.In);
                string sharedOutputFolder = IncarnationParams.IncarnatePath(node.DataFolders.ExchangeUrlFromResource, taskId, CopyPhase.Out);
                string tmpFolder = IncarnationParams.IncarnatePath(node.DataFolders.LocalFolder, taskId, CopyPhase.None);

                IOProxy.Ftp.MakePath(ftpFolder);
                IOProxy.Ftp.MakePath(jobFtpFolder);

                string jobFileName = "job_" + taskId + ".cmd";

                Log.Info(String.Format(
                    "Trying to exec task {0} on win PC {1}",
                    taskId, node.NodeName
                ));

                var pack = node.Packages.First(p => String.Equals(p.Name, incarnation.PackageName, StringComparison.InvariantCultureIgnoreCase));
                string batchContent = "";
                batchContent += "mkdir " + tmpFolder.TrimEnd(new char[] { '/', '\\' }) + Environment.NewLine;

                if (Path.IsPathRooted(tmpFolder)) // change drive if needed
                    batchContent += Path.GetPathRoot(tmpFolder).TrimEnd(new char[] { '/', '\\' }) + Environment.NewLine;

                batchContent += String.Format(
                    @"cd {0}" + Environment.NewLine,
                    tmpFolder.TrimEnd(new char[] { '/', '\\' })
                );

                batchContent += "echo %time% > clavire_script_started" + Environment.NewLine;

                foreach (string copyPath in pack.CopyOnStartup)
                    batchContent += String.Format(
                        @"xcopy {0} {1}\ /z /s /e /c /i /h /r /y" + Environment.NewLine,
                        copyPath.TrimEnd(new char[] { '/', '\\' }),
                        tmpFolder.TrimEnd(new char[] { '/', '\\' })
                    );

                batchContent += String.Format(
                    //@"ping localhost -w 1000 -n 50" + Environment.NewLine +
                    @"xcopy {0} {1}\ /z /s /e /c /i /h /r /y" + Environment.NewLine,
                    sharedInputFolder.TrimEnd(new char[] {'/', '\\'}),
                    tmpFolder.TrimEnd(new char[] { '/', '\\' })
                );

                // todo : env vars on WinPc provider

                string commandLine = incarnation.CommandLine;
                //var pack = node.Packages.First(p => commandLine.StartsWith(p.Name, StringComparison.InvariantCultureIgnoreCase));
                //commandLine = pack.Params["appPath"] + commandLine.Substring(pack.Name.Length);
                commandLine = String.Format(incarnation.CommandLine, pack.AppPath);
                //commandLine = String.Format(incarnation.CommandLine, pack.Params["appPath"]);

                batchContent += "echo %time% > clavire_task_started" + Environment.NewLine;
                batchContent += //"start \"" + jobFileName + " " + incarnation.PackageNameInConfig + "\" /wait /b" +
                    "cmd.exe /c " + commandLine + Environment.NewLine;
                batchContent += "echo %time% > clavire_task_finished" + Environment.NewLine;

                foreach (string delPath in pack.Cleanup)
                {
                    batchContent += String.Format(
                        @"rmdir /s /q {0}" + Environment.NewLine +
                        @"del /f /s /q {0}" + Environment.NewLine,
                        tmpFolder + delPath
                    );
                }

                batchContent += String.Format(
                    @"xcopy {1} {0}\ /z /s /e /c /i /h /r /y" + Environment.NewLine,
                    sharedOutputFolder.TrimEnd(new char[] { '/', '\\' }),
                    tmpFolder.TrimEnd(new char[] { '/', '\\' })
                );

                batchContent += String.Format(
                    @"ping localhost -n 3" + Environment.NewLine +
                    @"echo %time% > clavire_script_finished" + Environment.NewLine +
                    @"xcopy clavire_script_finished {1}\ /z /s /e /c /i /h /r /y" + Environment.NewLine +
                    @"cd {0}" + Environment.NewLine +
                    @"cd .." + Environment.NewLine +
                    //@"rmdir /s /q {0}" + Environment.NewLine +
                    "",
                    tmpFolder.TrimEnd(new char[] { '/', '\\' }),
                    sharedOutputFolder.TrimEnd(new char[] { '/', '\\' })
                );

                IOProxy.Ftp.UploadFileContent(batchContent, jobFtpFolder, jobFileName);

                //string cmdArgs = "/c " + CONST.Path.PsExec.Replace("PsExec.exe", "p.cmd");
                //string cmdArgs = "\\\\192.168.4.1 -u nano -p Yt1NyDpQNm -d cmd.exe /c \"\\\\192.168.4.1\\ftp_exchange\\Tasks\\10043\\job_10043.cmd\"";
                //Log.Debug(cmdArgs);
                //Process.Start(CONST.Path.PsExec, cmdArgs);

                //**/
                //var psexecProcess = new Process();
                //psexecProcess.StartInfo.UseShellExecute = false;
                ////psexecProcess.StartInfo.RedirectStandardOutput = true;
                ////psexecProcess.StartInfo.RedirectStandardError = true;
                //psexecProcess.StartInfo.FileName = CONST.Path.PsExec;
                //psexecProcess.StartInfo.Arguments = String.Format(
                //    "\\\\{0} -d -u {1} -p {2} cmd.exe /c {4}", // -d -w \"{3}\"  ^> C:\\Temp\\out
                //    //"-u nano -p Yt1NyDpQNm cmd.exe /c " + CONST.Path.PsExec.Replace("PsExec.exe", "p.cmd"),
                //    resParams.name, resParams.user, resParams.pass,
                //    resParams.tempFolderOnMachine.Replace(@"\", @"\\"),
                //    sharedJobFilePath
                //);

                //*
                //psexecProcess.StartInfo.UserName = "******";
                //psexecProcess.StartInfo.Password = new System.Security.SecureString();
                //foreach (var c in "Yt1NyDpQNm".ToCharArray())
                //{
                //    psexecProcess.StartInfo.Password.AppendChar(c);
                //}
                //**/

                //Log.Debug("psexec args:\n" + psexecProcess.StartInfo.Arguments);
                ////psexecProcess.Start();
                //Log.Debug("psexec process started");

                //string execMessage = /*psexecProcess.StandardOutput.ReadToEnd() + " " +*/ "1 " + PS_PID_START_MSG + "5."; //psexecProcess.StandardError.ReadToEnd();
                //execMessage = execMessage.Trim();
                ////psexecProcess.WaitForExit();
                //System.Threading.Thread.Sleep(3000);
                //Log.Debug("psexec output:\n" + execMessage);

                //if (!execMessage.Contains(PS_PID_START_MSG))
                //    throw new Exception(String.Format(
                //        "Couldn't exec task {0} on win pc {1}",
                //        taskId, resParams.name
                //    ));

                //execMessage = execMessage.Remove(0, execMessage.IndexOf(PS_PID_START_MSG) + PS_PID_START_MSG.Length);
                //string pid = execMessage.Substring(0, execMessage.Length-1);

                var rexService = EntryPointProxy.GetREx(node.Services.ExecutionUrl);
                int pid = rexService.Exec(taskId);

                Log.Debug(String.Format(
                    "Task {0} ({1}) started on pc {2} with pid = {3}",
                    taskId, pack.Name, node.NodeName, pid
                ));

                _nodeUsed[node.NodeName] = true;

                //System.Threading.Thread.Sleep(1000);

                return pid + "\n" + node.NodeName;
            }
        }
Exemplo n.º 14
0
        public override string Run(ulong taskId, IncarnationParams incarnation, Resource resource, IEnumerable<NodeConfig> nodesConfig)
        {
            string providedTaskId = taskId.ToString();

            var node = GetDefaultNodeSettings(resource, nodesConfig);
            var pack = node.PackageByName(incarnation.PackageName);

            var service = EntryPointProxy.GetClustersService();
            ClustersService.Code errCode;
            ClustersService.TaskInfo taskInfo;

            lock (_clustersServiceLock)
            {
                taskInfo = service.GetTaskState(providedTaskId, out errCode);
            }

            if (errCode != ServiceProxies.ClustersService.Code.OperationSuccess)
                throw new ClusterException(errCode);

            taskInfo.ClusterName = resource.ResourceName;
            taskInfo.CommandLine = String.Format(incarnation.CommandLine, pack.AppPath);
            taskInfo.PackageName = incarnation.PackageName.ToUpperInvariant();

            /*
            if (!String.IsNullOrEmpty(incarnation.StdInFile))
                taskInfo.StdinFileName = incarnation.StdInFile;
            else
                taskInfo.StdinFileName = "";

            if (!String.IsNullOrEmpty(incarnation.StdOutFile))
                taskInfo.StdoutFileName = incarnation.StdOutFile;
            else
                taskInfo.StdoutFileName = "";
            */

            // cores on nodes: {n, 0, 0} -> {n}
            taskInfo.NumberOfCores = new ClustersService.ArrayOfInt();
            taskInfo.NumberOfCores.AddRange(nodesConfig.Where(conf => conf.Cores > 0).Select(conf => conf.Cores));

            taskInfo.NumberOfNodes = taskInfo.NumberOfCores.Count;

            var logStream = new StringWriter();
            logStream.WriteLine("Задача {0} ({1}) запускается на кластере {2}",
                taskInfo.TaskID, taskInfo.PackageName, taskInfo.ClusterName);
            logStream.WriteLine("     Папка с файлами расчета: {0}", taskInfo.FTPPath);
            logStream.WriteLine("     Строка запуска: {0}", taskInfo.CommandLine);
            logStream.WriteLine("     Перенаправление вывода: {0}", taskInfo.StdoutFileName);
            logStream.Write("     Количество ядер (по каждому узлу): ");
            foreach (int coresCount in taskInfo.NumberOfCores)
                logStream.Write("{0} ", coresCount);

            Log.Info(logStream.ToString());

            lock (_clustersServiceLock)
            {
                errCode = service.ExecuteTask(taskInfo);
            }

            if (errCode != ServiceProxies.ClustersService.Code.OperationSuccess)
                throw new ClusterException(String.Format(
                    CONST.Dirty<string>("Ошибка интегратора управления кластерами при запуске задачи: {0}"), errCode.ToString()
                ));

            return providedTaskId;
        }
Exemplo n.º 15
0
        public override string Run(ulong taskId, IncarnationParams incarnation, Resource resource, IEnumerable <NodeConfig> nodesConfig)
        {
            lock (_hadoopLock)
            {
                var node = GetDefaultNodeSettings(resource, nodesConfig);
                var pack = node.PackageByName(incarnation.PackageName);

                string workDir                = IncarnationParams.IncarnatePath(node.DataFolders.LocalFolder, taskId, CopyPhase.None);
                string workScriptPath         = workDir + "hadoop.sh";
                string workScriptInternalPath = workDir + "hadoop_internal.sh";

                string exchangeInDir  = IncarnationParams.IncarnatePath(node.DataFolders.ExchangeUrlFromResource, taskId, CopyPhase.In);
                string exchangeOutDir = IncarnationParams.IncarnatePath(node.DataFolders.ExchangeUrlFromResource, taskId, CopyPhase.Out);

                string identity = exchangeInDir.Split('@')[0].Replace("ftp://", "");
                string user     = identity.Split(':')[0];
                string pass     = identity.Split(':')[1];

                string hostAndPath = exchangeInDir.Split('@')[1];

                string ftpHost   = hostAndPath.Split(new char[] { '/' }, 2)[0];
                string ftpInPath = hostAndPath.Split(new char[] { '/' }, 2)[1];

                string ftpOutPath = exchangeOutDir.Split('@')[1].Split(new char[] { '/' }, 2)[1];

                //pack.Params.

                var cmd = new StringBuilder();
                cmd.AppendFormat("cd " + workDir + "\n");
                cmd.AppendFormat(String.Format(HadoopCommands.Run, workScriptInternalPath) + "\n");
                cmd.AppendFormat("echo $!" + "\n");
                string tempPath = Path.GetTempFileName();
                File.WriteAllText(tempPath, cmd.ToString());

                var cmd_internal = new StringBuilder();
                cmd_internal.AppendFormat("ftp -n {0} << END_SCRIPT\n", ftpHost);
                cmd_internal.AppendFormat("quote User {0}\n", user);
                cmd_internal.AppendFormat("quote PASS {0}\n", pass);
                foreach (TaskFileDescription fileDesc in incarnation.FilesToCopy)
                {
                    cmd_internal.AppendFormat("get {0}{2} {1}{2}\n", ftpInPath, workDir, fileDesc.FileName);
                }
                cmd_internal.AppendFormat("quit" + "\n");
                cmd_internal.AppendFormat("END_SCRIPT" + "\n");
                cmd_internal.AppendFormat(String.Format(incarnation.CommandLine, IncarnationParams.IncarnatePath(pack.AppPath, taskId, CopyPhase.None).TrimEnd('/')) + "\n");
                cmd_internal.AppendFormat("ftp -n {0} << END_SCRIPT\n", ftpHost);
                cmd_internal.AppendFormat("quote User {0}\n", user);
                cmd_internal.AppendFormat("quote PASS {0}\n", pass);
                cmd_internal.AppendFormat("mkdir {0}\n", ftpOutPath);
                foreach (string fileName in incarnation.ExpectedOutputFileNames)
                {
                    cmd_internal.AppendFormat("put {0}{2} {1}{2}\n", workDir, ftpOutPath, fileName);
                }
                cmd_internal.AppendFormat("quit" + "\n");
                cmd_internal.AppendFormat("END_SCRIPT" + "\n");

                string tempPathInternal = Path.GetTempFileName();
                File.WriteAllText(tempPathInternal, cmd_internal.ToString());

                string host = node.Services.ExecutionUrl;

                SshRun(host, "mkdir " + workDir, true);                     // Need check, del
                SshRun(host, "rm " + workDir + "*", true);                  // Need check

                foreach (string path in pack.CopyOnStartup)                 // todo : internal_script maybe?
                {
                    string toCopy = (path.EndsWith("/")? path + "*": path); // 'folder/*' needed for contents copy
                    // ^^^^^ doesn't work everywhere

                    SshRun(host, "cp -fpR " + toCopy + "/* " + workDir);
                }

                /*
                 * SshRun(host, "cp " + "/home/clavire/hadrawler/clavire.sh " + workDir); // Need check, del
                 * SshRun(host, "cp " + "/home/clavire/hadrawler/hadoop2.conf " + workDir); // Need check, del
                 */

                ScpPut(host, tempPath, workScriptPath);// Need del
                SshRun(host, "chmod 700 " + workScriptPath);

                ScpPut(host, tempPathInternal, workScriptInternalPath);// Need del
                SshRun(host, "chmod 700 " + workScriptInternalPath);

                File.Delete(tempPath);
                File.Delete(tempPathInternal);

                string pid = SshShell(host, workScriptPath); // проверка на то запустилось ли

                _nodeUsed = true;
                return(pid.Split('\n')[2].TrimEnd('\r'));
            }
        }
Exemplo n.º 16
0
        public object Run(TaskRunContext task)
        {
            ulong taskId     = task.TaskId;
            int   coresToUse = (int)task.NodesConfig.Sum(cfg => cfg.Cores);
            var   node       = GetNode(task);

            string ftpFolder          = IncarnationParams.IncarnatePath(node.DataFolders.ExchangeUrlFromSystem, taskId, CopyPhase.In);
            string jobFtpFolder       = IncarnationParams.IncarnatePath(node.DataFolders.ExchangeUrlFromSystem, taskId, CopyPhase.None);
            string sharedInputFolder  = IncarnationParams.IncarnatePath(node.DataFolders.ExchangeUrlFromResource, taskId, CopyPhase.In);
            string sharedOutputFolder = IncarnationParams.IncarnatePath(node.DataFolders.ExchangeUrlFromResource, taskId, CopyPhase.Out);
            string tmpFolder          = IncarnationParams.IncarnatePath(node.DataFolders.LocalFolder, taskId, CopyPhase.None);

            IOProxy.Ftp.MakePath(ftpFolder);
            IOProxy.Ftp.MakePath(jobFtpFolder);

            string jobFileName = "job_" + taskId + ".cmd";

            logger.Info("Trying to exec task {0} on win PC {1}.{2}", taskId, node.ResourceName, node.NodeName);

            var    pack         = node.Packages.First(p => String.Equals(p.Name, task.PackageName, StringComparison.InvariantCultureIgnoreCase));
            string batchContent = "";

            batchContent += "mkdir " + tmpFolder.TrimEnd(new char[] { '/', '\\' }) + Environment.NewLine;

            if (Path.IsPathRooted(tmpFolder)) // change drive if needed
            {
                batchContent += Path.GetPathRoot(tmpFolder).TrimEnd(new char[] { '/', '\\' }) + Environment.NewLine;
            }

            batchContent += String.Format(
                @"cd {0}" + Environment.NewLine,
                tmpFolder.TrimEnd(new char[] { '/', '\\' })
                );

            batchContent += "echo %time% > clavire_script_started" + Environment.NewLine;

            foreach (string copyPath in pack.CopyOnStartup)
            {
                batchContent += String.Format(
                    @"xcopy {0} {1}\ /z /s /e /c /i /h /r /y" + Environment.NewLine,
                    copyPath.TrimEnd(new char[] { '/', '\\' }),
                    tmpFolder.TrimEnd(new char[] { '/', '\\' })
                    );
            }

            batchContent += String.Format(
                //@"ping localhost -w 1000 -n 50" + Environment.NewLine +
                @"xcopy {0} {1}\ /z /s /e /c /i /h /r /y" + Environment.NewLine,
                sharedInputFolder.TrimEnd(new char[] { '/', '\\' }),
                tmpFolder.TrimEnd(new char[] { '/', '\\' })
                );

            foreach (var envVar in pack.EnvVars)
            {
                batchContent += "set " + envVar.Key + "=" + envVar.Value + Environment.NewLine;
            }

            string commandLine = task.CommandLine;

            //var pack = node.Packages.First(p => commandLine.StartsWith(p.Name, StringComparison.InvariantCultureIgnoreCase));
            //commandLine = pack.Params["appPath"] + commandLine.Substring(pack.Name.Length);
            commandLine = String.Format(task.CommandLine, pack.AppPath);
            //commandLine = String.Format(incarnation.CommandLine, pack.Params["appPath"]);

            batchContent += "echo %time% > clavire_task_started" + Environment.NewLine;
            batchContent += //"start \"" + jobFileName + " " + incarnation.PackageNameInConfig + "\" /wait /b" +
                            "cmd.exe /c " + commandLine + Environment.NewLine;
            batchContent += "echo %time% > clavire_task_finished" + Environment.NewLine;


            foreach (string copyPath in pack.CleanupIgnore)
            {
                batchContent += String.Format(
                    @"xcopy {1} {0} /z /s /e /c /i /h /r /y" + Environment.NewLine,
                    (sharedOutputFolder.TrimEnd(new char[] { '/', '\\' }) + "/" + copyPath.TrimStart(new char[] { '/', '\\' })).Replace("/", "\\"),
                    (tmpFolder.TrimEnd(new char[] { '/', '\\' }) + "/" + copyPath.TrimStart(new char[] { '/', '\\' })).Replace("/", "\\")
                    );
            }

            foreach (string delPath in pack.Cleanup)
            {
                batchContent += String.Format(
                    @"rmdir /s /q {0}" + Environment.NewLine +
                    @"del /f /s /q {0}" + Environment.NewLine,
                    tmpFolder + delPath  // todo: delPath.TrimStart
                    );
            }

            batchContent += String.Format(
                @"xcopy {1} {0}\ /z /s /e /c /i /h /r /y" + Environment.NewLine,
                sharedOutputFolder.TrimEnd(new char[] { '/', '\\' }),
                tmpFolder.TrimEnd(new char[] { '/', '\\' })
                );

            batchContent += String.Format(
                //@"ping localhost -n 3" + Environment.NewLine +
                @"echo %time% > clavire_script_finished" + Environment.NewLine +
                @"xcopy clavire_script_finished {1}\ /z /s /e /c /i /h /r /y" + Environment.NewLine +
                @"cd {0}" + Environment.NewLine +
                @"cd .." + Environment.NewLine +
                //@"rmdir /s /q {0}" + Environment.NewLine +
                "",
                tmpFolder.TrimEnd(new char[] { '/', '\\' }),
                sharedOutputFolder.TrimEnd(new char[] { '/', '\\' })
                );


            int pauseLine = -1;

            Int32.TryParse(Config.AppSettings[DEBUG_PAUSE_PARAM_NAME] ?? "-1", out pauseLine);
            if (pauseLine >= 0)
            {
                var    batchLines      = batchContent.Replace("\r", "").Split(new[] { '\n' });
                string newBatchContent =
                    String.Join(Environment.NewLine, batchLines.Take(pauseLine)) + Environment.NewLine +
                    "pause" + Environment.NewLine +
                    String.Join(Environment.NewLine, batchLines.Skip(pauseLine));
                batchContent = newBatchContent;
            }


            IOProxy.Ftp.UploadFileContent(batchContent, jobFtpFolder, jobFileName);


            var rexService = GetREx(node.Services.ExecutionUrl);     // todo : close service client!
            int pid        = rexService.Exec(taskId);

            logger.Info("Task {0} ({1}) started on pc {2}.{3} with pid = {4}", taskId, pack.Name, node.ResourceName, node.NodeName, pid);

            return(pid + "\n" + node.NodeName);
        }
Exemplo n.º 17
0
 /// <summary>
 /// Run task on selected resource
 /// </summary>
 /// <param name="incarnation"></param>
 /// <param name="resource"></param>
 /// <returns>Provided task id (unique for resource)</returns>
 public abstract string Run(ulong taskId, IncarnationParams incarnation, Resource resource, IEnumerable <NodeConfig> nodesConfig);
Exemplo n.º 18
0
        public override string Run(ulong taskId, IncarnationParams incarnation, Resource resource, IEnumerable <NodeConfig> nodesConfig)
        {
            lock (_gridLock)
            {
                RefreshCertificate();

                string tmpFileName = null;
                if (incarnation.UserCert != null)
                {
                    Log.Info("Using user's certificate");
                    tmpFileName = Path.GetTempFileName();
                    IOProxy.Storage.Download(incarnation.UserCert, tmpFileName);

                    var scpForCert = new SSH.Scp(HELPER_SSH_HOST, HELPER_SSH_USER, HELPER_SSH_PASS);
                    scpForCert.Connect();
                    scpForCert.Recursive = true;
                    scpForCert.Put(tmpFileName, "/tmp/x509up_u500");
                    scpForCert.Close();

                    File.Delete(tmpFileName);
                    SshExec(PilotCommands.SetPermissionsOnProxyCertFile);
                }
                else
                {
                    Log.Info("Using system's certificate");
                }

                try
                {
                    int coresToUse = nodesConfig.Sum(conf => conf.Cores);
                    var node       = GetDefaultNodeSettings(resource, nodesConfig);
                    var pack       = node.PackageByName(incarnation.PackageName);

                    if (_nodeUsed[node.NodeName])
                    {
                        throw new Exception(String.Format("Could not run task {0} on node {1}: node used by another task", taskId, node.NodeName));
                    }

                    // todo : remove
                    incarnation.CommandLine = incarnation.CommandLine.Replace("java -jar ", "");
                    if (incarnation.PackageName.ToLowerInvariant() == "cnm")
                    {
                        incarnation.CommandLine = incarnation.CommandLine.Replace("{0}", "ru.ifmo.hpc.main.ExtendedModel");
                    }
                    else
                    if (incarnation.PackageName.ToLowerInvariant() == "ism")
                    {
                        incarnation.CommandLine = incarnation.CommandLine.Replace("{0}", "ru.ifmo.hpc.main.SpreadModel");
                    }
                    else
                    {
                        //if (incarnation.PackageName.ToLowerInvariant() == "orca")
                        incarnation.CommandLine = incarnation.CommandLine.Replace("{0}", "");
                    }


                    string ftpFolderFromSystem   = IncarnationParams.IncarnatePath(node.DataFolders.ExchangeUrlFromSystem, taskId, CopyPhase.In);
                    string ftpFolderFromResource = IncarnationParams.IncarnatePath(node.DataFolders.ExchangeUrlFromResource, taskId, CopyPhase.In);

                    string gridFtpFolder = IncarnationParams.IncarnatePath(node.DataFolders.LocalFolder, taskId, CopyPhase.None);
                    SshExec(PilotCommands.MakeFolderOnGridFtp, gridFtpFolder);

                    string endl = "\n";

                    // Сначала дописываем недостающий входной файл (скрипт запуска пакета на кластере)

                    string scriptName = pack.AppPath;

                    //if (pack.EnvVars.Any())
                    {
                        // Файл с установкой переменных окружения, если пакет их использует

                        scriptName = "run.sh";
                        var scriptContent = new StringBuilder();
                        scriptContent.Append("#!/bin/bash" + endl);
                        foreach (var pair in pack.EnvVars)
                        {
                            scriptContent.AppendFormat("export {0}={1}" + endl, pair.Key, pair.Value);
                        }

                        scriptContent.Append(pack.AppPath);

                        if (incarnation.PackageName.ToLowerInvariant() == "orca")
                        {
                            string[] args = incarnation.CommandLine.Split(new[] { ' ' }, StringSplitOptions.RemoveEmptyEntries);
                            for (int i = 0; i < args.Length; i++)
                            {
                                if (args[i] == "orca.out")
                                {
                                    scriptContent.Append(" >");
                                }

                                scriptContent.Append(" $" + (i + 1).ToString());
                            }
                        }
                        else
                        {
                            scriptContent.Append(" " + incarnation.CommandLine);
                        }

                        string scriptLocalPath = Path.GetTempFileName();
                        File.WriteAllText(scriptLocalPath, scriptContent.ToString());
                        IOProxy.Ftp.UploadLocalFile(scriptLocalPath, ftpFolderFromSystem, scriptName);

                        File.Delete(scriptLocalPath);
                    }

                    //IOProxy.Ftp.UploadLocalFile(DEFAULT_JOB_LAUNCHER_PATH, GetFtpInputFolder(taskId), Path.GetFileName(DEFAULT_JOB_LAUNCHER_PATH));

                    // Копируем входные файлы с ФТП на ГридФТП

                    SshExec(PilotCommands.CopyFilesToGridFtp, ftpFolderFromResource + " " + gridFtpFolder);
                    SshExec(PilotCommands.MakeFilesExecutableOnGridFtp, gridFtpFolder + "*");

                    // Формируем описание задания для грида

                    var jobFileContent = new StringBuilder();

                    jobFileContent.AppendFormat(@"{{ ""version"": 2, ""description"": ""{0}""," + endl, taskId);
                    jobFileContent.AppendFormat(@"  ""default_storage_base"": ""{0}""," + endl, gridFtpFolder);
                    jobFileContent.AppendFormat(@"  ""tasks"": [ {{ ""id"": ""a"", ""description"": ""task"", ""definition"": {{ ""version"": 2," + endl);
                    jobFileContent.AppendFormat(@"      ""executable"": ""{0}""," + endl, scriptName);
                    //jobFileContent.AppendFormat(@"      ""arguments"": [ ""{0}"" ]," + endl, String.Join(@""", """, args));

                    jobFileContent.AppendFormat(@"      ""input_files"": {{" + endl);
                    if (scriptName == "run.sh") // todo : if no input files?
                    {
                        jobFileContent.AppendFormat(@"          ""run.sh"": ""run.sh""," + endl);
                    }
                    jobFileContent.AppendFormat(@"          " + String.Join(
                                                    "," + endl + "          ",
                                                    incarnation.FilesToCopy.Select(
                                                        file => String.Format(@"""{0}"": ""{0}""", file.FileName)
                                                        )
                                                    ));
                    jobFileContent.AppendFormat(endl + @"      }}," + endl);

                    jobFileContent.AppendFormat(@"      ""output_files"": {{" + endl);

                    //if (incarnation.PackageName.ToLowerInvariant() == "cnm")
                    //    jobFileContent.AppendFormat(@"          ""output.dat"": ""output.dat""" + endl);
                    //else
                    if (incarnation.PackageName.ToLowerInvariant() == "ism")
                    {
                        jobFileContent.AppendFormat(@"          ""output.dat"": ""output.dat""" + endl);
                    }
                    else
                    if (incarnation.PackageName.ToLowerInvariant() == "orca")
                    {
                        jobFileContent.AppendFormat(@"          ""orca.out"":    ""orca.out""," + endl);
                        jobFileContent.AppendFormat(@"          ""eldens.cube"": ""eldens.cube""" + endl);
                    }
                    else
                    {
                        jobFileContent.AppendFormat(@"          " + String.Join(
                                                        "," + endl + "          ",
                                                        incarnation.ExpectedOutputFileNames
                                                        .Where(name => name != "std.out" && name != "std.err")
                                                        .Select(
                                                            name => String.Format(@"""{0}"": ""{0}""", name)
                                                            )
                                                        ) + endl);
                    }

                    jobFileContent.AppendFormat(@"      }}," + endl);

                    jobFileContent.AppendFormat(@"      ""stdout"": ""std.out"", ""stderr"": ""std.err"", " + endl);
                    jobFileContent.AppendFormat(@"      ""count"": {0}" + endl, coresToUse);

                    if (pack.Params.ContainsKey("requirements"))
                    {
                        jobFileContent.AppendFormat(@"      ,""requirements"": {0}" + endl, pack.Params["requirements"]);
                    }

                    jobFileContent.AppendFormat(@"  }} }} ]," + endl);

                    jobFileContent.AppendFormat(@"  ""requirements"": {{ ""hostname"": [""{0}""]", node.NodeAddress);

                    //if (pack.Params.ContainsKey("requirements"))
                    //    jobFileContent.AppendFormat(@", {0}" + endl, pack.Params["requirements"]);

                    jobFileContent.AppendFormat(@"}}" + endl + "}}", node.NodeAddress);

                    Log.Debug(String.Format("Task's '{0}' grid job JSON: ", taskId, jobFileContent));

                    string jobFileName         = "job_" + taskId.ToString() + ".js";
                    string jobFilePathOnHelper = JOBS_FOLDER_ON_HELPER + jobFileName;

                    //string jobFileContent = File.ReadAllText(DEFAULT_JOB_DESCR_PATH).Replace(GRIDFTP_PATH_TOKEN, taskFolderOnGridFtp);
                    string jobFilePathLocal = Path.GetTempFileName();
                    File.WriteAllText(jobFilePathLocal, jobFileContent.ToString());

                    // Записываем его на сервер с Пилотом

                    var scp = new SSH.Scp(HELPER_SSH_HOST, HELPER_SSH_USER, HELPER_SSH_PASS);

                    /*
                     * var notifier = new JobDescriptionUploadNotifier(TaskId, Cluster, RunParams);
                     * scp.OnTransferEnd += new SSH.FileTransferEvent(notifier.OnFinish); // todo : необязательно
                     */

                    scp.Connect();
                    scp.Recursive = true;
                    scp.Put(jobFilePathLocal, jobFilePathOnHelper);
                    scp.Close();

                    File.Delete(jobFilePathLocal); // todo : remove files on helper and gridftp

                    // Запускаем

                    Log.Info(String.Format(
                                 "Trying to exec task {0} on grid cluster {1}",
                                 taskId, node.NodeName
                                 ));

                    string launchResult = SshExec(PilotCommands.SubmitJob, jobFilePathOnHelper, pilotUrl: node.Services.ExecutionUrl);
                    int    urlPos       = launchResult.IndexOf("https://");
                    string jobUrl       = launchResult.Substring(urlPos).Trim() + "a";
                    Log.Debug(jobUrl);

                    Log.Info(String.Format(
                                 "Task {0} launched on grid with jobUrl = {1}",
                                 taskId, jobUrl
                                 ));

                    _nodeUsed[node.NodeName] = true;

                    return(jobUrl);
                }
                catch (Exception e)
                {
                    Log.Error(String.Format(
                                  "Error while starting task {0} in grid: {1}\n{2}",
                                  taskId, e.Message, e.StackTrace
                                  ));

                    throw;
                }
                finally
                {
                    if (incarnation.UserCert != null)
                    {
                        Log.Info("Wiping user's certificate");
                        tmpFileName = Path.GetTempFileName();
                        File.WriteAllText(tmpFileName, "Wiped by Easis system");

                        var scpForCert = new SSH.Scp(HELPER_SSH_HOST, HELPER_SSH_USER, HELPER_SSH_PASS);
                        scpForCert.Connect();
                        scpForCert.Recursive = true;
                        scpForCert.Put(tmpFileName, "/tmp/x509up_u500");
                        scpForCert.Close();

                        File.Delete(tmpFileName);
                        SshExec(PilotCommands.SetPermissionsOnProxyCertFile);
                    }
                }
            }
        }
Exemplo n.º 19
0
        public override string Run(ulong taskId, IncarnationParams incarnation, Resource resource, IEnumerable<NodeConfig> nodesConfig)
        {
            lock (_gridLock)
            {
                RefreshCertificate();

                string tmpFileName = null;
                if (incarnation.UserCert != null)
                {
                    Log.Info("Using user's certificate");
                    tmpFileName = Path.GetTempFileName();
                    IOProxy.Storage.Download(incarnation.UserCert, tmpFileName);

                    var scpForCert = new SSH.Scp(HELPER_SSH_HOST, HELPER_SSH_USER, HELPER_SSH_PASS);
                    scpForCert.Connect();
                    scpForCert.Recursive = true;
                    scpForCert.Put(tmpFileName, "/tmp/x509up_u500");
                    scpForCert.Close();

                    File.Delete(tmpFileName);
                    SshExec(PilotCommands.SetPermissionsOnProxyCertFile);
                }
                else
                {
                    Log.Info("Using system's certificate");
                }

                try
                {
                    int coresToUse = nodesConfig.Sum(conf => conf.Cores);
                    var node = GetDefaultNodeSettings(resource, nodesConfig);
                    var pack = node.PackageByName(incarnation.PackageName);

                    if (_nodeUsed[node.NodeName])
                        throw new Exception(String.Format("Could not run task {0} on node {1}: node used by another task", taskId, node.NodeName));

                    // todo : remove
                    incarnation.CommandLine = incarnation.CommandLine.Replace("java -jar ", "");
                    if (incarnation.PackageName.ToLowerInvariant() == "cnm")
                        incarnation.CommandLine = incarnation.CommandLine.Replace("{0}", "ru.ifmo.hpc.main.ExtendedModel");
                    else
                    if (incarnation.PackageName.ToLowerInvariant() == "ism")
                        incarnation.CommandLine = incarnation.CommandLine.Replace("{0}", "ru.ifmo.hpc.main.SpreadModel");
                    else
                    //if (incarnation.PackageName.ToLowerInvariant() == "orca")
                        incarnation.CommandLine = incarnation.CommandLine.Replace("{0}", "");


                    string ftpFolderFromSystem = IncarnationParams.IncarnatePath(node.DataFolders.ExchangeUrlFromSystem, taskId, CopyPhase.In);
                    string ftpFolderFromResource = IncarnationParams.IncarnatePath(node.DataFolders.ExchangeUrlFromResource, taskId, CopyPhase.In);

                    string gridFtpFolder = IncarnationParams.IncarnatePath(node.DataFolders.LocalFolder, taskId, CopyPhase.None);
                    SshExec(PilotCommands.MakeFolderOnGridFtp, gridFtpFolder);

                    string endl = "\n";

                    // Сначала дописываем недостающий входной файл (скрипт запуска пакета на кластере)

                    string scriptName = pack.AppPath;

                    //if (pack.EnvVars.Any())
                    {
                        // Файл с установкой переменных окружения, если пакет их использует

                        scriptName = "run.sh";
                        var scriptContent = new StringBuilder();
                        scriptContent.Append("#!/bin/bash" + endl);
                        foreach (var pair in pack.EnvVars)
                            scriptContent.AppendFormat("export {0}={1}" + endl, pair.Key, pair.Value);
                        
                        scriptContent.Append(pack.AppPath);

                        if (incarnation.PackageName.ToLowerInvariant() == "orca")
                        {
                            string[] args = incarnation.CommandLine.Split(new[] { ' ' }, StringSplitOptions.RemoveEmptyEntries);
                            for (int i = 0; i < args.Length; i++)
                            {
                                if (args[i] == "orca.out")
                                    scriptContent.Append(" >");

                                scriptContent.Append(" $" + (i + 1).ToString());
                            }
                        }
                        else
                        {
                            scriptContent.Append(" " + incarnation.CommandLine);
                        }

                        string scriptLocalPath = Path.GetTempFileName();
                        File.WriteAllText(scriptLocalPath, scriptContent.ToString());
                        IOProxy.Ftp.UploadLocalFile(scriptLocalPath, ftpFolderFromSystem, scriptName);

                        File.Delete(scriptLocalPath);
                    }

                    //IOProxy.Ftp.UploadLocalFile(DEFAULT_JOB_LAUNCHER_PATH, GetFtpInputFolder(taskId), Path.GetFileName(DEFAULT_JOB_LAUNCHER_PATH));

                    // Копируем входные файлы с ФТП на ГридФТП

                    SshExec(PilotCommands.CopyFilesToGridFtp, ftpFolderFromResource + " " + gridFtpFolder);
                    SshExec(PilotCommands.MakeFilesExecutableOnGridFtp, gridFtpFolder + "*");

                    // Формируем описание задания для грида

                    var jobFileContent = new StringBuilder();

                    jobFileContent.AppendFormat(@"{{ ""version"": 2, ""description"": ""{0}""," + endl, taskId);
                    jobFileContent.AppendFormat(@"  ""default_storage_base"": ""{0}""," + endl, gridFtpFolder);
                    jobFileContent.AppendFormat(@"  ""tasks"": [ {{ ""id"": ""a"", ""description"": ""task"", ""definition"": {{ ""version"": 2," + endl);
                    jobFileContent.AppendFormat(@"      ""executable"": ""{0}""," + endl, scriptName);
                    //jobFileContent.AppendFormat(@"      ""arguments"": [ ""{0}"" ]," + endl, String.Join(@""", """, args));

                    jobFileContent.AppendFormat(@"      ""input_files"": {{" + endl);
                    if (scriptName == "run.sh") // todo : if no input files?
                        jobFileContent.AppendFormat(@"          ""run.sh"": ""run.sh""," + endl);
                    jobFileContent.AppendFormat(@"          " + String.Join(
                        "," + endl + "          ",
                        incarnation.FilesToCopy.Select(
                            file => String.Format(@"""{0}"": ""{0}""", file.FileName)
                        )
                    ));
                    jobFileContent.AppendFormat(endl + @"      }}," + endl);

                    jobFileContent.AppendFormat(@"      ""output_files"": {{" + endl);

                    //if (incarnation.PackageName.ToLowerInvariant() == "cnm")
                    //    jobFileContent.AppendFormat(@"          ""output.dat"": ""output.dat""" + endl);
                    //else
                    if (incarnation.PackageName.ToLowerInvariant() == "ism")
                        jobFileContent.AppendFormat(@"          ""output.dat"": ""output.dat""" + endl);
                    else
                    if (incarnation.PackageName.ToLowerInvariant() == "orca")
                    {
                        jobFileContent.AppendFormat(@"          ""orca.out"":    ""orca.out""," + endl);
                        jobFileContent.AppendFormat(@"          ""eldens.cube"": ""eldens.cube""" + endl);
                    }
                    else
                    {
                        jobFileContent.AppendFormat(@"          " + String.Join(
                            "," + endl + "          ",
                            incarnation.ExpectedOutputFileNames
                                .Where(name => name != "std.out" && name != "std.err")
                                .Select(
                                    name => String.Format(@"""{0}"": ""{0}""", name)
                                )
                        ) + endl);
                    }

                    jobFileContent.AppendFormat(@"      }}," + endl);

                    jobFileContent.AppendFormat(@"      ""stdout"": ""std.out"", ""stderr"": ""std.err"", " + endl);
                    jobFileContent.AppendFormat(@"      ""count"": {0}" + endl, coresToUse);

                    if (pack.Params.ContainsKey("requirements"))
                        jobFileContent.AppendFormat(@"      ,""requirements"": {0}" + endl, pack.Params["requirements"]);

                    jobFileContent.AppendFormat(@"  }} }} ]," + endl);

                    jobFileContent.AppendFormat(@"  ""requirements"": {{ ""hostname"": [""{0}""]", node.NodeAddress);

                    //if (pack.Params.ContainsKey("requirements"))
                    //    jobFileContent.AppendFormat(@", {0}" + endl, pack.Params["requirements"]);

                    jobFileContent.AppendFormat(@"}}" + endl + "}}", node.NodeAddress);

                    Log.Debug(String.Format("Task's '{0}' grid job JSON: ", taskId, jobFileContent));

                    string jobFileName = "job_" + taskId.ToString() + ".js";
                    string jobFilePathOnHelper = JOBS_FOLDER_ON_HELPER + jobFileName;

                    //string jobFileContent = File.ReadAllText(DEFAULT_JOB_DESCR_PATH).Replace(GRIDFTP_PATH_TOKEN, taskFolderOnGridFtp);
                    string jobFilePathLocal = Path.GetTempFileName();
                    File.WriteAllText(jobFilePathLocal, jobFileContent.ToString());

                    // Записываем его на сервер с Пилотом

                    var scp = new SSH.Scp(HELPER_SSH_HOST, HELPER_SSH_USER, HELPER_SSH_PASS);

                    /*
                    var notifier = new JobDescriptionUploadNotifier(TaskId, Cluster, RunParams);
                    scp.OnTransferEnd += new SSH.FileTransferEvent(notifier.OnFinish); // todo : необязательно
                    */

                    scp.Connect();
                    scp.Recursive = true;
                    scp.Put(jobFilePathLocal, jobFilePathOnHelper);
                    scp.Close();

                    File.Delete(jobFilePathLocal); // todo : remove files on helper and gridftp

                    // Запускаем

                    Log.Info(String.Format(
                        "Trying to exec task {0} on grid cluster {1}",
                        taskId, node.NodeName
                    ));

                    string launchResult = SshExec(PilotCommands.SubmitJob, jobFilePathOnHelper, pilotUrl: node.Services.ExecutionUrl);
                    int urlPos = launchResult.IndexOf("https://");
                    string jobUrl = launchResult.Substring(urlPos).Trim() + "a";
                    Log.Debug(jobUrl);

                    Log.Info(String.Format(
                        "Task {0} launched on grid with jobUrl = {1}",
                        taskId, jobUrl
                    ));

                    _nodeUsed[node.NodeName] = true;

                    return jobUrl;
                }
                catch (Exception e)
                {
                    Log.Error(String.Format(
                        "Error while starting task {0} in grid: {1}\n{2}",
                        taskId, e.Message, e.StackTrace
                    ));

                    throw;
                }
                finally
                {
                    if (incarnation.UserCert != null)
                    {
                        Log.Info("Wiping user's certificate");
                        tmpFileName = Path.GetTempFileName();
                        File.WriteAllText(tmpFileName, "Wiped by Easis system");

                        var scpForCert = new SSH.Scp(HELPER_SSH_HOST, HELPER_SSH_USER, HELPER_SSH_PASS);
                        scpForCert.Connect();
                        scpForCert.Recursive = true;
                        scpForCert.Put(tmpFileName, "/tmp/x509up_u500");
                        scpForCert.Close();

                        File.Delete(tmpFileName);
                        SshExec(PilotCommands.SetPermissionsOnProxyCertFile);
                    }
                }
            }
        }
Exemplo n.º 20
0
 /// <summary>
 /// Run task on selected resource
 /// </summary>
 /// <param name="incarnation"></param>
 /// <param name="resource"></param>
 /// <returns>Provided task id (unique for resource)</returns>
 public abstract string Run(ulong taskId, IncarnationParams incarnation, Resource resource, IEnumerable<NodeConfig> nodesConfig);
Exemplo n.º 21
0
        public override string Run(ulong taskId, IncarnationParams incarnation, Resource resource, IEnumerable <NodeConfig> nodesConfig)
        {
            string providedTaskId = taskId.ToString();

            var node = GetDefaultNodeSettings(resource, nodesConfig);
            var pack = node.PackageByName(incarnation.PackageName);

            var service = EntryPointProxy.GetClustersService();

            ClustersService.Code     errCode;
            ClustersService.TaskInfo taskInfo;

            lock (_clustersServiceLock)
            {
                taskInfo = service.GetTaskState(providedTaskId, out errCode);
            }

            if (errCode != ServiceProxies.ClustersService.Code.OperationSuccess)
            {
                throw new ClusterException(errCode);
            }

            taskInfo.ClusterName = resource.ResourceName;
            taskInfo.CommandLine = String.Format(incarnation.CommandLine, pack.AppPath);
            taskInfo.PackageName = incarnation.PackageName.ToUpperInvariant();

            /*
             * if (!String.IsNullOrEmpty(incarnation.StdInFile))
             *  taskInfo.StdinFileName = incarnation.StdInFile;
             * else
             *  taskInfo.StdinFileName = "";
             *
             * if (!String.IsNullOrEmpty(incarnation.StdOutFile))
             *  taskInfo.StdoutFileName = incarnation.StdOutFile;
             * else
             *  taskInfo.StdoutFileName = "";
             */

            // cores on nodes: {n, 0, 0} -> {n}
            taskInfo.NumberOfCores = new ClustersService.ArrayOfInt();
            taskInfo.NumberOfCores.AddRange(nodesConfig.Where(conf => conf.Cores > 0).Select(conf => conf.Cores));

            taskInfo.NumberOfNodes = taskInfo.NumberOfCores.Count;

            var logStream = new StringWriter();

            logStream.WriteLine("Задача {0} ({1}) запускается на кластере {2}",
                                taskInfo.TaskID, taskInfo.PackageName, taskInfo.ClusterName);
            logStream.WriteLine("     Папка с файлами расчета: {0}", taskInfo.FTPPath);
            logStream.WriteLine("     Строка запуска: {0}", taskInfo.CommandLine);
            logStream.WriteLine("     Перенаправление вывода: {0}", taskInfo.StdoutFileName);
            logStream.Write("     Количество ядер (по каждому узлу): ");
            foreach (int coresCount in taskInfo.NumberOfCores)
            {
                logStream.Write("{0} ", coresCount);
            }

            Log.Info(logStream.ToString());

            lock (_clustersServiceLock)
            {
                errCode = service.ExecuteTask(taskInfo);
            }

            if (errCode != ServiceProxies.ClustersService.Code.OperationSuccess)
            {
                throw new ClusterException(String.Format(
                                               CONST.Dirty <string>("Ошибка интегратора управления кластерами при запуске задачи: {0}"), errCode.ToString()
                                               ));
            }

            return(providedTaskId);
        }
Exemplo n.º 22
0
        public override string Run(ulong taskId, IncarnationParams incarnation, Resource resource, IEnumerable<NodeConfig> nodesConfig)
        {
            lock (_hadoopLock)
            {
                var node = GetDefaultNodeSettings(resource, nodesConfig);
                var pack = node.PackageByName(incarnation.PackageName);

                string workDir = IncarnationParams.IncarnatePath(node.DataFolders.LocalFolder, taskId, CopyPhase.None);
                string workScriptPath = workDir + "hadoop.sh";
                string workScriptInternalPath = workDir + "hadoop_internal.sh";

                string exchangeInDir = IncarnationParams.IncarnatePath(node.DataFolders.ExchangeUrlFromResource, taskId, CopyPhase.In);
                string exchangeOutDir = IncarnationParams.IncarnatePath(node.DataFolders.ExchangeUrlFromResource, taskId, CopyPhase.Out);

                string identity = exchangeInDir.Split('@')[0].Replace("ftp://", "");
                string user = identity.Split(':')[0];
                string pass = identity.Split(':')[1];

                string hostAndPath = exchangeInDir.Split('@')[1];

                string ftpHost = hostAndPath.Split(new char[] { '/' }, 2)[0];
                string ftpInPath = hostAndPath.Split(new char[] { '/' }, 2)[1];

                string ftpOutPath = exchangeOutDir.Split('@')[1].Split(new char[] { '/' }, 2)[1];

                //pack.Params.

                var cmd = new StringBuilder();
                cmd.AppendFormat("cd " + workDir + "\n");
                cmd.AppendFormat(String.Format(HadoopCommands.Run, workScriptInternalPath) + "\n");
                cmd.AppendFormat("echo $!" + "\n");
                string tempPath = Path.GetTempFileName();
                File.WriteAllText(tempPath, cmd.ToString());

                var cmd_internal = new StringBuilder();
                cmd_internal.AppendFormat("ftp -n {0} << END_SCRIPT\n", ftpHost);
                cmd_internal.AppendFormat("quote User {0}\n", user);
                cmd_internal.AppendFormat("quote PASS {0}\n", pass);
                foreach (TaskFileDescription fileDesc in incarnation.FilesToCopy)
                {
                    cmd_internal.AppendFormat("get {0}{2} {1}{2}\n", ftpInPath, workDir, fileDesc.FileName);
                }
                cmd_internal.AppendFormat("quit" + "\n");
                cmd_internal.AppendFormat("END_SCRIPT" + "\n");
                cmd_internal.AppendFormat(String.Format(incarnation.CommandLine, IncarnationParams.IncarnatePath(pack.AppPath, taskId, CopyPhase.None).TrimEnd('/')) + "\n");
                cmd_internal.AppendFormat("ftp -n {0} << END_SCRIPT\n", ftpHost);
                cmd_internal.AppendFormat("quote User {0}\n", user);
                cmd_internal.AppendFormat("quote PASS {0}\n", pass);
                cmd_internal.AppendFormat("mkdir {0}\n", ftpOutPath);
                foreach (string fileName in incarnation.ExpectedOutputFileNames)
                {
                    cmd_internal.AppendFormat("put {0}{2} {1}{2}\n", workDir, ftpOutPath, fileName);
                }
                cmd_internal.AppendFormat("quit" + "\n");
                cmd_internal.AppendFormat("END_SCRIPT" + "\n");

                string tempPathInternal = Path.GetTempFileName();
                File.WriteAllText(tempPathInternal, cmd_internal.ToString());

                string host = node.Services.ExecutionUrl;

                SshRun(host, "mkdir " + workDir, true); // Need check, del
                SshRun(host, "rm " + workDir + "*", true); // Need check

                foreach (string path in pack.CopyOnStartup) // todo : internal_script maybe?
                {
                    string toCopy = (path.EndsWith("/")? path + "*": path); // 'folder/*' needed for contents copy
                    // ^^^^^ doesn't work everywhere

                    SshRun(host, "cp -fpR " + toCopy + "/* " + workDir);
                }

                /*
                SshRun(host, "cp " + "/home/clavire/hadrawler/clavire.sh " + workDir); // Need check, del
                SshRun(host, "cp " + "/home/clavire/hadrawler/hadoop2.conf " + workDir); // Need check, del
                */

                ScpPut(host, tempPath, workScriptPath);// Need del
                SshRun(host, "chmod 700 " + workScriptPath);

                ScpPut(host, tempPathInternal, workScriptInternalPath);// Need del
                SshRun(host, "chmod 700 " + workScriptInternalPath);

                File.Delete(tempPath);
                File.Delete(tempPathInternal);

                string pid = SshShell(host, workScriptPath); // проверка на то запустилось ли

                _nodeUsed = true;
                return pid.Split('\n')[2].TrimEnd('\r');
            }
        }
Exemplo n.º 23
0
        public override string Run(ulong taskId, IncarnationParams incarnation, Resource resource, IEnumerable <NodeConfig> nodesConfig)
        {
            lock (_pcLock)
            {
                //AcceptPsToolsEula();

                int coresToUse = nodesConfig.Sum(conf => conf.Cores);
                var node       = GetDefaultNodeSettings(resource, nodesConfig);

                if (_nodeUsed[node.NodeName])
                {
                    throw new Exception(String.Format("Could not run task {0} on node {1}: node used by another task", taskId, node.NodeName));
                }

                string ftpFolder          = IncarnationParams.IncarnatePath(node.DataFolders.ExchangeUrlFromSystem, taskId, CopyPhase.In);
                string jobFtpFolder       = IncarnationParams.IncarnatePath(node.DataFolders.ExchangeUrlFromSystem, taskId, CopyPhase.None);
                string sharedInputFolder  = IncarnationParams.IncarnatePath(node.DataFolders.ExchangeUrlFromResource, taskId, CopyPhase.In);
                string sharedOutputFolder = IncarnationParams.IncarnatePath(node.DataFolders.ExchangeUrlFromResource, taskId, CopyPhase.Out);
                string tmpFolder          = IncarnationParams.IncarnatePath(node.DataFolders.LocalFolder, taskId, CopyPhase.None);

                IOProxy.Ftp.MakePath(ftpFolder);
                IOProxy.Ftp.MakePath(jobFtpFolder);

                string jobFileName = "job_" + taskId + ".cmd";

                Log.Info(String.Format(
                             "Trying to exec task {0} on win PC {1}",
                             taskId, node.NodeName
                             ));

                var    pack         = node.Packages.First(p => String.Equals(p.Name, incarnation.PackageName, StringComparison.InvariantCultureIgnoreCase));
                string batchContent = "";
                batchContent += "mkdir " + tmpFolder.TrimEnd(new char[] { '/', '\\' }) + Environment.NewLine;

                if (Path.IsPathRooted(tmpFolder)) // change drive if needed
                {
                    batchContent += Path.GetPathRoot(tmpFolder).TrimEnd(new char[] { '/', '\\' }) + Environment.NewLine;
                }

                batchContent += String.Format(
                    @"cd {0}" + Environment.NewLine,
                    tmpFolder.TrimEnd(new char[] { '/', '\\' })
                    );

                batchContent += "echo %time% > clavire_script_started" + Environment.NewLine;

                foreach (string copyPath in pack.CopyOnStartup)
                {
                    batchContent += String.Format(
                        @"xcopy {0} {1}\ /z /s /e /c /i /h /r /y" + Environment.NewLine,
                        copyPath.TrimEnd(new char[] { '/', '\\' }),
                        tmpFolder.TrimEnd(new char[] { '/', '\\' })
                        );
                }

                batchContent += String.Format(
                    //@"ping localhost -w 1000 -n 50" + Environment.NewLine +
                    @"xcopy {0} {1}\ /z /s /e /c /i /h /r /y" + Environment.NewLine,
                    sharedInputFolder.TrimEnd(new char[] { '/', '\\' }),
                    tmpFolder.TrimEnd(new char[] { '/', '\\' })
                    );

                // todo : env vars on WinPc provider

                string commandLine = incarnation.CommandLine;
                //var pack = node.Packages.First(p => commandLine.StartsWith(p.Name, StringComparison.InvariantCultureIgnoreCase));
                //commandLine = pack.Params["appPath"] + commandLine.Substring(pack.Name.Length);
                commandLine = String.Format(incarnation.CommandLine, pack.AppPath);
                //commandLine = String.Format(incarnation.CommandLine, pack.Params["appPath"]);

                batchContent += "echo %time% > clavire_task_started" + Environment.NewLine;
                batchContent += //"start \"" + jobFileName + " " + incarnation.PackageNameInConfig + "\" /wait /b" +
                                "cmd.exe /c " + commandLine + Environment.NewLine;
                batchContent += "echo %time% > clavire_task_finished" + Environment.NewLine;

                foreach (string delPath in pack.Cleanup)
                {
                    batchContent += String.Format(
                        @"rmdir /s /q {0}" + Environment.NewLine +
                        @"del /f /s /q {0}" + Environment.NewLine,
                        tmpFolder + delPath
                        );
                }


                batchContent += String.Format(
                    @"xcopy {1} {0}\ /z /s /e /c /i /h /r /y" + Environment.NewLine,
                    sharedOutputFolder.TrimEnd(new char[] { '/', '\\' }),
                    tmpFolder.TrimEnd(new char[] { '/', '\\' })
                    );

                batchContent += String.Format(
                    @"ping localhost -n 3" + Environment.NewLine +
                    @"echo %time% > clavire_script_finished" + Environment.NewLine +
                    @"xcopy clavire_script_finished {1}\ /z /s /e /c /i /h /r /y" + Environment.NewLine +
                    @"cd {0}" + Environment.NewLine +
                    @"cd .." + Environment.NewLine +
                    //@"rmdir /s /q {0}" + Environment.NewLine +
                    "",
                    tmpFolder.TrimEnd(new char[] { '/', '\\' }),
                    sharedOutputFolder.TrimEnd(new char[] { '/', '\\' })
                    );

                IOProxy.Ftp.UploadFileContent(batchContent, jobFtpFolder, jobFileName);

                //string cmdArgs = "/c " + CONST.Path.PsExec.Replace("PsExec.exe", "p.cmd");
                //string cmdArgs = "\\\\192.168.4.1 -u nano -p Yt1NyDpQNm -d cmd.exe /c \"\\\\192.168.4.1\\ftp_exchange\\Tasks\\10043\\job_10043.cmd\"";
                //Log.Debug(cmdArgs);
                //Process.Start(CONST.Path.PsExec, cmdArgs);

                //**/
                //var psexecProcess = new Process();
                //psexecProcess.StartInfo.UseShellExecute = false;
                ////psexecProcess.StartInfo.RedirectStandardOutput = true;
                ////psexecProcess.StartInfo.RedirectStandardError = true;
                //psexecProcess.StartInfo.FileName = CONST.Path.PsExec;
                //psexecProcess.StartInfo.Arguments = String.Format(
                //    "\\\\{0} -d -u {1} -p {2} cmd.exe /c {4}", // -d -w \"{3}\"  ^> C:\\Temp\\out
                //    //"-u nano -p Yt1NyDpQNm cmd.exe /c " + CONST.Path.PsExec.Replace("PsExec.exe", "p.cmd"),
                //    resParams.name, resParams.user, resParams.pass,
                //    resParams.tempFolderOnMachine.Replace(@"\", @"\\"),
                //    sharedJobFilePath
                //);

                //*
                //psexecProcess.StartInfo.UserName = "******";
                //psexecProcess.StartInfo.Password = new System.Security.SecureString();
                //foreach (var c in "Yt1NyDpQNm".ToCharArray())
                //{
                //    psexecProcess.StartInfo.Password.AppendChar(c);
                //}
                //**/

                //Log.Debug("psexec args:\n" + psexecProcess.StartInfo.Arguments);
                ////psexecProcess.Start();
                //Log.Debug("psexec process started");

                //string execMessage = /*psexecProcess.StandardOutput.ReadToEnd() + " " +*/ "1 " + PS_PID_START_MSG + "5."; //psexecProcess.StandardError.ReadToEnd();
                //execMessage = execMessage.Trim();
                ////psexecProcess.WaitForExit();
                //System.Threading.Thread.Sleep(3000);
                //Log.Debug("psexec output:\n" + execMessage);

                //if (!execMessage.Contains(PS_PID_START_MSG))
                //    throw new Exception(String.Format(
                //        "Couldn't exec task {0} on win pc {1}",
                //        taskId, resParams.name
                //    ));

                //execMessage = execMessage.Remove(0, execMessage.IndexOf(PS_PID_START_MSG) + PS_PID_START_MSG.Length);
                //string pid = execMessage.Substring(0, execMessage.Length-1);

                var rexService = EntryPointProxy.GetREx(node.Services.ExecutionUrl);
                int pid        = rexService.Exec(taskId);

                Log.Debug(String.Format(
                              "Task {0} ({1}) started on pc {2} with pid = {3}",
                              taskId, pack.Name, node.NodeName, pid
                              ));

                _nodeUsed[node.NodeName] = true;

                //System.Threading.Thread.Sleep(1000);

                return(pid + "\n" + node.NodeName);
            }
        }