public static Dictionary<NodeConfig, Estimation> GetEstimationsByModel(PackageEngineState engineState, IEnumerable<Resource> resources, IEnumerable<ResourceNode> permittedNodes) { var estims = new Dictionary<NodeConfig, Estimation>(); Log.Debug("Estimating by model..."); var engine = new PackageEngine(engineState.CompiledDef, engineState.EngineContext); if (!engine.CanEstimate()) { Log.Debug("Can't estimate by model"); return estims; } try { var allRes = resources.Select(r => new Common.Resource() { Name = r.ResourceName, Nodes = r.Nodes.Select(n => new Common.Node() { ResourceName = n.ResourceName, DNSName = n.NodeName, Parameters = new Dictionary<string, string>(n.StaticHardwareParams), CoresAvailable = n.CoresAvailable, CoresTotal = (int)n.CoresCount, }).ToArray(), Parameters = new Dictionary<string, string>(r.HardwareParams), }); Log.Debug("Permitted nodes: " + String.Join(", ", permittedNodes.Select(n => n.ResourceName + "." + n.NodeName))); foreach (var node in permittedNodes) { try { var res = allRes.Single(r => r.Name == node.ResourceName); var dest = new Common.LaunchDestination() { ResourceName = node.ResourceName, NodeNames = new[] { node.NodeName }, }; var modelExecParams = new Dictionary<string, object>(); foreach (string paramId in TimeMeter.ClusterParameterReader.GetAvailableParameterIds()) // from scheduler. Why? { try { modelExecParams[paramId] = TimeMeter.ClusterParameterReader.GetValue(paramId, res, dest); } catch (Exception) { /* it's ok not to extract all possible params */ } } var modelCoefs = GetModelCoefs(engine, node); var modelEstimation = engine.Estimate(modelExecParams, modelCoefs); if (engine.Ctx.Result.Messages.Any()) { Log.Warn("Messages on estimation: " + String.Join("\n", engine.Ctx.Result.Messages.Select(mess => mess.Message))); } //double estimationInSeconds = engine.Estimate(modelExecParams, modelCoefs); if (modelEstimation != null /* == no errors */ && modelEstimation.CalculationTime.IsSet && !Double.IsInfinity(modelEstimation.CalculationTime.Value) && !Double.IsNaN(modelEstimation.CalculationTime.Value)) { var modelCoeffsToRemember = new Dictionary<string, double>(); foreach (var pair in modelCoefs) { if (pair.Value is double) modelCoeffsToRemember[pair.Key] = (double)pair.Value; } estims.Add ( new NodeConfig() { ResourceName = node.ResourceName, NodeName = node.NodeName, Cores = (uint) 1, // was node.CoresAvailable // todo : node.pack.MinCores or node.pack.MaxCores // todo : estimation from model -> cores }, new Estimation(modelEstimation) { ModelCoeffs = modelCoeffsToRemember } ); /* Log.Debug(String.Format("Estim by model on {0}.{1}: {2}", node.ResourceName, node.NodeName, (modelEstimation.CalculationTime.IsSet ? modelEstimation.CalculationTime.Value.ToString() : "not set") )); */ } else { // todo : else Log.Trace estimation by model is NaN or Infinity Log.Warn("Model estimation failed for task " + engineState._taskDescription.TaskId.ToString()); } } catch (Exception estimEx) { Log.Warn(String.Format( "Exception while estimating task {1} on node '{2}' by models in PackageBase : {0}", estimEx, engineState._taskDescription.TaskId, node.NodeName )); } } } catch (Exception e) { Log.Error(String.Format( "Exception while getting estimations from models in PackageBase for task {2}: {0}\n{1}", e.Message, e.StackTrace, engineState._taskDescription.TaskId )); } return estims; }
public static Dictionary <NodeConfig, Estimation> GetEstimationsByModel(PackageEngineState engineState, IEnumerable <Resource> resources, IEnumerable <ResourceNode> permittedNodes) { var estims = new Dictionary <NodeConfig, Estimation>(); Log.Debug("Estimating by model..."); var engine = new PackageEngine(engineState.CompiledDef, engineState.EngineContext); if (!engine.CanEstimate()) { Log.Debug("Can't estimate by model"); return(estims); } try { var allRes = resources.Select(r => new Common.Resource() { Name = r.ResourceName, Nodes = r.Nodes.Select(n => new Common.Node() { ResourceName = n.ResourceName, DNSName = n.NodeName, Parameters = new Dictionary <string, string>(n.StaticHardwareParams), CoresAvailable = n.CoresAvailable, CoresTotal = (int)n.CoresCount, }).ToArray(), Parameters = new Dictionary <string, string>(r.HardwareParams), }); Log.Debug("Permitted nodes: " + String.Join(", ", permittedNodes.Select(n => n.ResourceName + "." + n.NodeName))); foreach (var node in permittedNodes) { try { var res = allRes.Single(r => r.Name == node.ResourceName); var dest = new Common.LaunchDestination() { ResourceName = node.ResourceName, NodeNames = new[] { node.NodeName }, }; var modelExecParams = new Dictionary <string, object>(); foreach (string paramId in TimeMeter.ClusterParameterReader.GetAvailableParameterIds()) // from scheduler. Why? { try { modelExecParams[paramId] = TimeMeter.ClusterParameterReader.GetValue(paramId, res, dest); } catch (Exception) { /* it's ok not to extract all possible params */ } } var modelCoefs = GetModelCoefs(engine, node); var modelEstimation = engine.Estimate(modelExecParams, modelCoefs); if (engine.Ctx.Result.Messages.Any()) { Log.Warn("Messages on estimation: " + String.Join("\n", engine.Ctx.Result.Messages.Select(mess => mess.Message))); } //double estimationInSeconds = engine.Estimate(modelExecParams, modelCoefs); if (modelEstimation != null /* == no errors */ && modelEstimation.CalculationTime.IsSet && !Double.IsInfinity(modelEstimation.CalculationTime.Value) && !Double.IsNaN(modelEstimation.CalculationTime.Value)) { var modelCoeffsToRemember = new Dictionary <string, double>(); foreach (var pair in modelCoefs) { if (pair.Value is double) { modelCoeffsToRemember[pair.Key] = (double)pair.Value; } } estims.Add ( new NodeConfig() { ResourceName = node.ResourceName, NodeName = node.NodeName, Cores = (uint)1, // was node.CoresAvailable // todo : node.pack.MinCores or node.pack.MaxCores // todo : estimation from model -> cores }, new Estimation(modelEstimation) { ModelCoeffs = modelCoeffsToRemember } ); /* * Log.Debug(String.Format("Estim by model on {0}.{1}: {2}", * node.ResourceName, node.NodeName, * (modelEstimation.CalculationTime.IsSet ? modelEstimation.CalculationTime.Value.ToString() : "not set") * )); */ } else { // todo : else Log.Trace estimation by model is NaN or Infinity Log.Warn("Model estimation failed for task " + engineState._taskDescription.TaskId.ToString()); } } catch (Exception estimEx) { Log.Warn(String.Format( "Exception while estimating task {1} on node '{2}' by models in PackageBase : {0}", estimEx, engineState._taskDescription.TaskId, node.NodeName )); } } } catch (Exception e) { Log.Error(String.Format( "Exception while getting estimations from models in PackageBase for task {2}: {0}\n{1}", e.Message, e.StackTrace, engineState._taskDescription.TaskId )); } return(estims); }
public EstimationResult Estimate(IDictionary <string, string> parameters, Common.Resource resource, Common.LaunchDestination destination, bool optimize) { var models = new List <IModel>(); models.AddRange(_models.Values); models.Sort(new ModelOrderer()); EstimationResult result = null; foreach (var model in models) { var hwParameters = model.GetParameters(ParameterSourceType.Hardware); foreach (var hwParameter in hwParameters) { model.SetParameterValue(hwParameter.Name, ClusterParameterReader.GetValue(hwParameter.Name, resource, destination)); } result = model.Estimate(parameters, result, optimize); } if (!result.Parameters.Exists(p => p.Name == NodesCountExtractor.NODES)) { result.Parameters.Add(new EstimationResult.ParameterValue() { Name = NodesCountExtractor.NODES, InitialValue = (1).ToString(), NewValue = (1).ToString() }); } if (!result.Parameters.Exists(p => p.Name == ProcessorCountPerNode.P)) { result.Parameters.Add(new EstimationResult.ParameterValue() { Name = ProcessorCountPerNode.P, InitialValue = (1).ToString(), NewValue = (1).ToString() }); } return(result); }