public static IEnumerable <string> ListExpectedOutputs(PackageEngineState engineState, out bool hasGroups) { var fileList = new List <string>(); hasGroups = false; var engine = new PackageEngine(engineState.CompiledDef, engineState.EngineContext); var expectedOutFilesAndGroups = engine.GetEnabledOutDefs() .Where(def => def is IOutFileDef || def is OutFileGroupDefBase) .Where(def => engine.Ctx.Memory[def.Name].IsRequiredOrExpected); if (expectedOutFilesAndGroups.Any(def => def is OutFileGroupDefBase)) { hasGroups = true; } foreach (var def in expectedOutFilesAndGroups) { if (def is IOutFileDef) { var fileDef = def as IOutFileDef; string filePath = (fileDef.Path.Trim(new[] { '/', '\\' }) + "/" + fileDef.ExpectedName) .Trim(new[] { '/', '\\' }) ; fileList.Add(filePath); } } return(fileList); }
public static IEnumerable<string> ListExpectedOutputs(PackageEngineState engineState, out bool hasGroups) { var fileList = new List<string>(); hasGroups = false; var engine = new PackageEngine(engineState.CompiledDef, engineState.EngineContext); var expectedOutFilesAndGroups = engine.GetEnabledOutDefs() .Where(def => def is IOutFileDef || def is OutFileGroupDefBase) .Where(def => engine.Ctx.Memory[def.Name].IsRequiredOrExpected); if (expectedOutFilesAndGroups.Any(def => def is OutFileGroupDefBase)) hasGroups = true; foreach (var def in expectedOutFilesAndGroups) { if (def is IOutFileDef) { var fileDef = def as IOutFileDef; string filePath = (fileDef.Path.Trim(new[] { '/', '\\' }) + "/" + fileDef.ExpectedName) .Trim(new[] { '/', '\\' }) ; fileList.Add(filePath); } } return fileList; }
public static IncarnationParams ProcessInputFiles(/* ref */ PackageEngineState engineState, out TimeSpan inputFilesTime) { var engine = new PackageEngine(engineState.CompiledDef, engineState.EngineContext); var incarnation = new IncarnationParams(); var fileManager = new InputFileManager(engineState.StoragePathBase); inputFilesTime = TimeSpanExt.Measure(() => { TimeSpan timeLimit = PB_PROCESS_TIME_LIMIT; try { Tools.ProcessWithTimeLimit(timeLimit, () => { engine.PrepareInputs(fileManager); CheckEngineResult(engine); }); } catch (TimeLimitException tle) { throw new PackageBaseException("Inputs processing in Package base exceeded time limit (" + timeLimit.TotalSeconds + " seconds).", tle); } finally { incarnation.FilesToCopy = incarnation.FilesToCopy.Concat(fileManager.GetInputFiles()).ToArray(); fileManager.Cleanup(); } }); object cmdLine = engine.Ctx.Memory[CmdLineDef.DefName].Value; //if (cmdLine == null) //throw new PackageBaseException("CommandLine generated by PackageBase is empty"); incarnation.CommandLine = cmdLine.ToString(); Log.Debug(String.Format( "Command line for pack {0}.{1} is '{2}' (task {3}). Cmdline object from PB is {4}", engine.CompiledMode.ModeQName.PackageName ?? "''", engine.CompiledMode.ModeQName.ModeName ?? "''", incarnation.CommandLine ?? "", engineState._taskDescription.TaskId, cmdLine == null ? "null" : "not null" )); if (String.IsNullOrWhiteSpace(incarnation.CommandLine)) { throw new PackageBaseException("CommandLine generated by PackageBase is empty"); } if (!engine.IsReadyToRun()) { Log.Warn("PackageBase: task is not ready to run after PrepareInputs"); Log.Error("PackageBase: task is not ready to run after PrepareInputs"); throw new PackageBaseException("PackageBase: task is not ready to run after processing inputs"); } return(incarnation); }
public static IncarnationParams ProcessInputFiles(/* ref */ PackageEngineState engineState, out TimeSpan inputFilesTime) { var engine = new PackageEngine(engineState.CompiledDef, engineState.EngineContext); var incarnation = new IncarnationParams(); var fileManager = new InputFileManager(engineState.StoragePathBase); inputFilesTime = TimeSpanExt.Measure(() => { TimeSpan timeLimit = PB_PROCESS_TIME_LIMIT; try { Tools.ProcessWithTimeLimit(timeLimit, () => { engine.PrepareInputs(fileManager); CheckEngineResult(engine); }); } catch (TimeLimitException tle) { throw new PackageBaseException("Inputs processing in Package base exceeded time limit (" + timeLimit.TotalSeconds + " seconds).", tle); } finally { incarnation.FilesToCopy = incarnation.FilesToCopy.Concat(fileManager.GetInputFiles()).ToArray(); fileManager.Cleanup(); } }); object cmdLine = engine.Ctx.Memory[CmdLineDef.DefName].Value; //if (cmdLine == null) //throw new PackageBaseException("CommandLine generated by PackageBase is empty"); incarnation.CommandLine = cmdLine.ToString(); Log.Debug(String.Format( "Command line for pack {0}.{1} is '{2}' (task {3}). Cmdline object from PB is {4}", engine.CompiledMode.ModeQName.PackageName ?? "''", engine.CompiledMode.ModeQName.ModeName ?? "''", incarnation.CommandLine ?? "", engineState._taskDescription.TaskId, cmdLine == null ? "null" : "not null" )); if (String.IsNullOrWhiteSpace(incarnation.CommandLine)) throw new PackageBaseException("CommandLine generated by PackageBase is empty"); if (!engine.IsReadyToRun()) { Log.Warn("PackageBase: task is not ready to run after PrepareInputs"); Log.Error("PackageBase: task is not ready to run after PrepareInputs"); throw new PackageBaseException("PackageBase: task is not ready to run after processing inputs"); } return incarnation; }
public static IDictionary <string, string> UpdateInputs( /* ref */ PackageEngineState engineState, IEnumerable <TaskFileDescription> moreFiles) { var fileParams = moreFiles.ToDictionary( f => f.SlotName, f => (f.FileName ?? f.SlotName) + "|" + f.StorageId ); return(UpdateInputs(engineState, fileParams)); }
public object Clone() { var clonedState = new PackageEngineState(); /**/ if (this.CompiledDef != null) clonedState.CompiledDef = this.CompiledDef; // immutable //clonedState.CompiledDef = (CompiledModeDef) this.CompiledDef.Clone(); /**/ if (this.EngineContext != null) clonedState.EngineContext = (PackageEngineContext) this.EngineContext.Clone(); clonedState.Init(this._taskDescription, this.StoragePathBase); return clonedState; }
public static IDictionary <string, string> UpdateInputs( /* ref */ PackageEngineState engineState, IDictionary <string, string> moreParams) { var engine = new PackageEngine(engineState.CompiledDef, engineState.EngineContext); engine.UpdateContractParams(moreParams); var processedInputs = engine.GetEnabledInsValues().Where(inp => inp.IsSet); var newInputParams = engine.Stringify(processedInputs, engine.CompiledMode.Defs.Values); //var newInputParams = processedInputs.ToDictionary( // inp => inp.VarName, // inp => (inp.Value == null)? null: inp.Value.ToString() //); return(newInputParams); }
public object Clone() { var clonedState = new PackageEngineState(); /**/ if (this.CompiledDef != null) { clonedState.CompiledDef = this.CompiledDef; // immutable } //clonedState.CompiledDef = (CompiledModeDef) this.CompiledDef.Clone(); /**/ if (this.EngineContext != null) { clonedState.EngineContext = (PackageEngineContext)this.EngineContext.Clone(); } clonedState.Init(this._taskDescription, this.StoragePathBase); return(clonedState); }
public static IDictionary<string, string> UpdateInputs( /* ref */ PackageEngineState engineState, IDictionary<string, string> moreParams) { var engine = new PackageEngine(engineState.CompiledDef, engineState.EngineContext); engine.UpdateContractParams(moreParams); var processedInputs = engine.GetEnabledInsValues().Where(inp => inp.IsSet); var newInputParams = engine.Stringify(processedInputs, engine.CompiledMode.Defs.Values); //var newInputParams = processedInputs.ToDictionary( // inp => inp.VarName, // inp => (inp.Value == null)? null: inp.Value.ToString() //); return newInputParams; }
public static IDictionary<string, string> UpdateInputs( /* ref */ PackageEngineState engineState, IEnumerable<TaskFileDescription> moreFiles) { var fileParams = moreFiles.ToDictionary( f => f.SlotName, f => (f.FileName ?? f.SlotName) + "|" + f.StorageId ); return UpdateInputs(engineState, fileParams); }
public static Dictionary<string, string> ProcessOutputs(PackageEngineState engineState, string ftpRoot, out IEnumerable<TaskFileDescription> outFiles, out TimeSpan outputFilesTime) { var forbiddenFileNamesRx = (new string[] { @"\.\./", @"torque", @"\.dataList\.src$", @"\.hosts$", @"\.sh$", @"\.exe$" }).Select(st => new Regex(st, RegexOptions.IgnoreCase)); string ftpRootWithoutSlash = ftpRoot.TrimEnd(new[] { '/', '\\' }); var allFilePaths = IOProxy.Ftp.GetFileNamesInAllTree(ftpRootWithoutSlash); var filteredFilePathsWithRoot = allFilePaths.Where(st => !forbiddenFileNamesRx.Any(rx => rx.IsMatch(st))); var filteredFilePaths = filteredFilePathsWithRoot.Select(st => st.Substring(ftpRoot.Length)); var fileManager = new OutputFileManager(ftpRoot, engineState.StoragePathBase); var engine = new PackageEngine(engineState.CompiledDef, engineState.EngineContext); var actionStarted = DateTime.Now; { TimeSpan timeLimit = PB_PROCESS_TIME_LIMIT; try { Tools.ProcessWithTimeLimit(timeLimit, () => { //engine.ProcessOutputs(outputFileNames, fileManager); engine.ProcessOutputs(filteredFilePaths, fileManager); }); } catch (TimeLimitException tle) { throw new PackageBaseException("Outputs processing in Package base exceeded time limit (" + timeLimit.TotalSeconds + " seconds).", tle); } finally { outFiles = fileManager.GetOutputFiles(); // includes Cleanup(); } } var actionFinished = DateTime.Now; outputFilesTime = actionFinished - actionStarted; // todo : fix ugliness var processedOuts = engine.GetEnabledOutsValues(); var outFilesAndGroups = engine.GetEnabledOutDefs().Where(def => def is IOutFileDef || def is OutFileGroupDefBase); foreach (var def in outFilesAndGroups) { var memoryEntry = processedOuts.First(me => me.VarName == def.Name); if (memoryEntry.IsSet) // file was there { if (def is IOutFileDef) { var extFileDef = memoryEntry.Value as ExternalFileDef; extFileDef.Locator = outFiles.FirstOrDefault(fd => fd.SlotName == def.Name).StorageId; if (extFileDef.Locator == null) // todo : null.StorageId will fail earlier { Log.Warn("No output file processed for param " + memoryEntry.VarName + ", and memoryEntry.IsSet == true"); } } else if (def is OutFileGroupDefBase) { var group = def as OutFileGroupDefBase; var extFileDefs = memoryEntry.Value as IEnumerable<ExternalFileDef>; foreach (var extFileDef in extFileDefs) { extFileDef.Locator = outFiles.FirstOrDefault(file => file.SlotName == def.Name && file.FileName == extFileDef.FileName ).StorageId; if (extFileDef.Locator == null) // todo : null.StorageId will fail earlier { Log.Warn(String.Format( "File '{0}' wasn't processed for param {1}, and memoryEntry.IsSet == true", extFileDef.FileName, memoryEntry.VarName )); } } } } } //var outputParams = engine.Stringify(processedOuts, engine.CompiledMode.Outs.Values); var outputParams = PackageBase.Serializer.PackageBaseSerializer.Serialize(processedOuts, engine.CompiledMode.Outs.Values); Log.Debug(String.Format("Task's {0} output params are: {1}", engineState._taskDescription.TaskId, String.Join(", ", outputParams.Select(pair => pair.Key + " = " + pair.Value)) )); return outputParams; }
public static Dictionary <string, string> ProcessOutputs(PackageEngineState engineState, string ftpRoot, out IEnumerable <TaskFileDescription> outFiles, out TimeSpan outputFilesTime) { var forbiddenFileNamesRx = (new string[] { @"\.\./", @"torque", @"\.dataList\.src$", @"\.hosts$", @"\.sh$", @"\.exe$" }).Select(st => new Regex(st, RegexOptions.IgnoreCase)); string ftpRootWithoutSlash = ftpRoot.TrimEnd(new[] { '/', '\\' }); var allFilePaths = IOProxy.Ftp.GetFileNamesInAllTree(ftpRootWithoutSlash); var filteredFilePathsWithRoot = allFilePaths.Where(st => !forbiddenFileNamesRx.Any(rx => rx.IsMatch(st))); var filteredFilePaths = filteredFilePathsWithRoot.Select(st => st.Substring(ftpRoot.Length)); var fileManager = new OutputFileManager(ftpRoot, engineState.StoragePathBase); var engine = new PackageEngine(engineState.CompiledDef, engineState.EngineContext); var actionStarted = DateTime.Now; { TimeSpan timeLimit = PB_PROCESS_TIME_LIMIT; try { Tools.ProcessWithTimeLimit(timeLimit, () => { //engine.ProcessOutputs(outputFileNames, fileManager); engine.ProcessOutputs(filteredFilePaths, fileManager); }); } catch (TimeLimitException tle) { throw new PackageBaseException("Outputs processing in Package base exceeded time limit (" + timeLimit.TotalSeconds + " seconds).", tle); } finally { outFiles = fileManager.GetOutputFiles(); // includes Cleanup(); } } var actionFinished = DateTime.Now; outputFilesTime = actionFinished - actionStarted; // todo : fix ugliness var processedOuts = engine.GetEnabledOutsValues(); var outFilesAndGroups = engine.GetEnabledOutDefs().Where(def => def is IOutFileDef || def is OutFileGroupDefBase); foreach (var def in outFilesAndGroups) { var memoryEntry = processedOuts.First(me => me.VarName == def.Name); if (memoryEntry.IsSet) // file was there { if (def is IOutFileDef) { var extFileDef = memoryEntry.Value as ExternalFileDef; extFileDef.Locator = outFiles.FirstOrDefault(fd => fd.SlotName == def.Name).StorageId; if (extFileDef.Locator == null) // todo : null.StorageId will fail earlier { Log.Warn("No output file processed for param " + memoryEntry.VarName + ", and memoryEntry.IsSet == true"); } } else if (def is OutFileGroupDefBase) { var group = def as OutFileGroupDefBase; var extFileDefs = memoryEntry.Value as IEnumerable <ExternalFileDef>; foreach (var extFileDef in extFileDefs) { extFileDef.Locator = outFiles.FirstOrDefault(file => file.SlotName == def.Name && file.FileName == extFileDef.FileName ).StorageId; if (extFileDef.Locator == null) // todo : null.StorageId will fail earlier { Log.Warn(String.Format( "File '{0}' wasn't processed for param {1}, and memoryEntry.IsSet == true", extFileDef.FileName, memoryEntry.VarName )); } } } } } //var outputParams = engine.Stringify(processedOuts, engine.CompiledMode.Outs.Values); var outputParams = PackageBase.Serializer.PackageBaseSerializer.Serialize(processedOuts, engine.CompiledMode.Outs.Values); Log.Debug(String.Format("Task's {0} output params are: {1}", engineState._taskDescription.TaskId, String.Join(", ", outputParams.Select(pair => pair.Key + " = " + pair.Value)) )); return(outputParams); }
public static Dictionary <NodeConfig, Estimation> GetEstimationsByModel(PackageEngineState engineState, IEnumerable <Resource> resources, IEnumerable <ResourceNode> permittedNodes) { var estims = new Dictionary <NodeConfig, Estimation>(); Log.Debug("Estimating by model..."); var engine = new PackageEngine(engineState.CompiledDef, engineState.EngineContext); if (!engine.CanEstimate()) { Log.Debug("Can't estimate by model"); return(estims); } try { var allRes = resources.Select(r => new Common.Resource() { Name = r.ResourceName, Nodes = r.Nodes.Select(n => new Common.Node() { ResourceName = n.ResourceName, DNSName = n.NodeName, Parameters = new Dictionary <string, string>(n.StaticHardwareParams), CoresAvailable = n.CoresAvailable, CoresTotal = (int)n.CoresCount, }).ToArray(), Parameters = new Dictionary <string, string>(r.HardwareParams), }); Log.Debug("Permitted nodes: " + String.Join(", ", permittedNodes.Select(n => n.ResourceName + "." + n.NodeName))); foreach (var node in permittedNodes) { try { var res = allRes.Single(r => r.Name == node.ResourceName); var dest = new Common.LaunchDestination() { ResourceName = node.ResourceName, NodeNames = new[] { node.NodeName }, }; var modelExecParams = new Dictionary <string, object>(); foreach (string paramId in TimeMeter.ClusterParameterReader.GetAvailableParameterIds()) // from scheduler. Why? { try { modelExecParams[paramId] = TimeMeter.ClusterParameterReader.GetValue(paramId, res, dest); } catch (Exception) { /* it's ok not to extract all possible params */ } } var modelCoefs = GetModelCoefs(engine, node); var modelEstimation = engine.Estimate(modelExecParams, modelCoefs); if (engine.Ctx.Result.Messages.Any()) { Log.Warn("Messages on estimation: " + String.Join("\n", engine.Ctx.Result.Messages.Select(mess => mess.Message))); } //double estimationInSeconds = engine.Estimate(modelExecParams, modelCoefs); if (modelEstimation != null /* == no errors */ && modelEstimation.CalculationTime.IsSet && !Double.IsInfinity(modelEstimation.CalculationTime.Value) && !Double.IsNaN(modelEstimation.CalculationTime.Value)) { var modelCoeffsToRemember = new Dictionary <string, double>(); foreach (var pair in modelCoefs) { if (pair.Value is double) { modelCoeffsToRemember[pair.Key] = (double)pair.Value; } } estims.Add ( new NodeConfig() { ResourceName = node.ResourceName, NodeName = node.NodeName, Cores = (uint)1, // was node.CoresAvailable // todo : node.pack.MinCores or node.pack.MaxCores // todo : estimation from model -> cores }, new Estimation(modelEstimation) { ModelCoeffs = modelCoeffsToRemember } ); /* * Log.Debug(String.Format("Estim by model on {0}.{1}: {2}", * node.ResourceName, node.NodeName, * (modelEstimation.CalculationTime.IsSet ? modelEstimation.CalculationTime.Value.ToString() : "not set") * )); */ } else { // todo : else Log.Trace estimation by model is NaN or Infinity Log.Warn("Model estimation failed for task " + engineState._taskDescription.TaskId.ToString()); } } catch (Exception estimEx) { Log.Warn(String.Format( "Exception while estimating task {1} on node '{2}' by models in PackageBase : {0}", estimEx, engineState._taskDescription.TaskId, node.NodeName )); } } } catch (Exception e) { Log.Error(String.Format( "Exception while getting estimations from models in PackageBase for task {2}: {0}\n{1}", e.Message, e.StackTrace, engineState._taskDescription.TaskId )); } return(estims); }
public static Dictionary<NodeConfig, Estimation> GetEstimationsByModel(PackageEngineState engineState, IEnumerable<Resource> resources, IEnumerable<ResourceNode> permittedNodes) { var estims = new Dictionary<NodeConfig, Estimation>(); Log.Debug("Estimating by model..."); var engine = new PackageEngine(engineState.CompiledDef, engineState.EngineContext); if (!engine.CanEstimate()) { Log.Debug("Can't estimate by model"); return estims; } try { var allRes = resources.Select(r => new Common.Resource() { Name = r.ResourceName, Nodes = r.Nodes.Select(n => new Common.Node() { ResourceName = n.ResourceName, DNSName = n.NodeName, Parameters = new Dictionary<string, string>(n.StaticHardwareParams), CoresAvailable = n.CoresAvailable, CoresTotal = (int)n.CoresCount, }).ToArray(), Parameters = new Dictionary<string, string>(r.HardwareParams), }); Log.Debug("Permitted nodes: " + String.Join(", ", permittedNodes.Select(n => n.ResourceName + "." + n.NodeName))); foreach (var node in permittedNodes) { try { var res = allRes.Single(r => r.Name == node.ResourceName); var dest = new Common.LaunchDestination() { ResourceName = node.ResourceName, NodeNames = new[] { node.NodeName }, }; var modelExecParams = new Dictionary<string, object>(); foreach (string paramId in TimeMeter.ClusterParameterReader.GetAvailableParameterIds()) // from scheduler. Why? { try { modelExecParams[paramId] = TimeMeter.ClusterParameterReader.GetValue(paramId, res, dest); } catch (Exception) { /* it's ok not to extract all possible params */ } } var modelCoefs = GetModelCoefs(engine, node); var modelEstimation = engine.Estimate(modelExecParams, modelCoefs); if (engine.Ctx.Result.Messages.Any()) { Log.Warn("Messages on estimation: " + String.Join("\n", engine.Ctx.Result.Messages.Select(mess => mess.Message))); } //double estimationInSeconds = engine.Estimate(modelExecParams, modelCoefs); if (modelEstimation != null /* == no errors */ && modelEstimation.CalculationTime.IsSet && !Double.IsInfinity(modelEstimation.CalculationTime.Value) && !Double.IsNaN(modelEstimation.CalculationTime.Value)) { var modelCoeffsToRemember = new Dictionary<string, double>(); foreach (var pair in modelCoefs) { if (pair.Value is double) modelCoeffsToRemember[pair.Key] = (double)pair.Value; } estims.Add ( new NodeConfig() { ResourceName = node.ResourceName, NodeName = node.NodeName, Cores = (uint) 1, // was node.CoresAvailable // todo : node.pack.MinCores or node.pack.MaxCores // todo : estimation from model -> cores }, new Estimation(modelEstimation) { ModelCoeffs = modelCoeffsToRemember } ); /* Log.Debug(String.Format("Estim by model on {0}.{1}: {2}", node.ResourceName, node.NodeName, (modelEstimation.CalculationTime.IsSet ? modelEstimation.CalculationTime.Value.ToString() : "not set") )); */ } else { // todo : else Log.Trace estimation by model is NaN or Infinity Log.Warn("Model estimation failed for task " + engineState._taskDescription.TaskId.ToString()); } } catch (Exception estimEx) { Log.Warn(String.Format( "Exception while estimating task {1} on node '{2}' by models in PackageBase : {0}", estimEx, engineState._taskDescription.TaskId, node.NodeName )); } } } catch (Exception e) { Log.Error(String.Format( "Exception while getting estimations from models in PackageBase for task {2}: {0}\n{1}", e.Message, e.StackTrace, engineState._taskDescription.TaskId )); } return estims; }