private static Dictionary <string, double> AutoAdjustCoefsByHistory(PackageEngine engine, ResourceNode node, Dictionary <string, object> fixedCoefs, Dictionary <string, double> adjustableCoefs)
{
lock (_historyLock)
{
//Log.Info("Adjusting coefs");
string packageName = engine.CompiledMode.ModeQName.PackageName.ToLowerInvariant();
var history = GetHistorySamples(packageName, node.ResourceName, node.NodeName);
if (history.Length < 1)
{
//Log.Debug("Not enough history samples to adjust coefs");
return(null);
}
if (packageName == "bsm")
{
double[] fTime = history.Select(hs => double.Parse(hs.PackParams["ForecastSize"])).ToArray();
double[] cTime = history.Select(hs => hs.CalcTime.TotalSeconds).ToArray();
double perf = BSMSimpleEstimator.BsmEstimatePerf(fTime, cTime);
double dev = BSMSimpleEstimator.BsmRelativeStDev(perf, fTime, cTime);
return(new Dictionary <string, double>()
{
{ "Perf", perf },
{ "D", dev }
});
}
else
{
Log.Warn("Unsupported package to adjust coefs: " + packageName);
return(null);
}
}
}
public static IEnumerable<string> ListExpectedOutputs(PackageEngineState engineState, out bool hasGroups)
{
var fileList = new List<string>();
hasGroups = false;
var engine = new PackageEngine(engineState.CompiledDef, engineState.EngineContext);
var expectedOutFilesAndGroups = engine.GetEnabledOutDefs()
.Where(def => def is IOutFileDef || def is OutFileGroupDefBase)
.Where(def => engine.Ctx.Memory[def.Name].IsRequiredOrExpected);
if (expectedOutFilesAndGroups.Any(def => def is OutFileGroupDefBase))
hasGroups = true;
foreach (var def in expectedOutFilesAndGroups)
{
if (def is IOutFileDef)
{
var fileDef = def as IOutFileDef;
string filePath =
(fileDef.Path.Trim(new[] { '/', '\\' }) + "/" + fileDef.ExpectedName)
.Trim(new[] { '/', '\\' })
;
fileList.Add(filePath);
}
}
return fileList;
}
public static IEnumerable <string> ListExpectedOutputs(PackageEngineState engineState, out bool hasGroups)
{
var fileList = new List <string>();
hasGroups = false;
var engine = new PackageEngine(engineState.CompiledDef, engineState.EngineContext);
var expectedOutFilesAndGroups = engine.GetEnabledOutDefs()
.Where(def => def is IOutFileDef || def is OutFileGroupDefBase)
.Where(def => engine.Ctx.Memory[def.Name].IsRequiredOrExpected);
if (expectedOutFilesAndGroups.Any(def => def is OutFileGroupDefBase))
{
hasGroups = true;
}
foreach (var def in expectedOutFilesAndGroups)
{
if (def is IOutFileDef)
{
var fileDef = def as IOutFileDef;
string filePath =
(fileDef.Path.Trim(new[] { '/', '\\' }) + "/" + fileDef.ExpectedName)
.Trim(new[] { '/', '\\' })
;
fileList.Add(filePath);
}
}
return(fileList);
}
public static IncarnationParams ProcessInputFiles(/* ref */ PackageEngineState engineState, out TimeSpan inputFilesTime)
{
var engine = new PackageEngine(engineState.CompiledDef, engineState.EngineContext);
var incarnation = new IncarnationParams();
var fileManager = new InputFileManager(engineState.StoragePathBase);
inputFilesTime = TimeSpanExt.Measure(() =>
{
TimeSpan timeLimit = PB_PROCESS_TIME_LIMIT;
try
{
Tools.ProcessWithTimeLimit(timeLimit, () =>
{
engine.PrepareInputs(fileManager);
CheckEngineResult(engine);
});
}
catch (TimeLimitException tle)
{
throw new PackageBaseException("Inputs processing in Package base exceeded time limit (" + timeLimit.TotalSeconds + " seconds).", tle);
}
finally
{
incarnation.FilesToCopy = incarnation.FilesToCopy.Concat(fileManager.GetInputFiles()).ToArray();
fileManager.Cleanup();
}
});
object cmdLine = engine.Ctx.Memory[CmdLineDef.DefName].Value;
//if (cmdLine == null)
//throw new PackageBaseException("CommandLine generated by PackageBase is empty");
incarnation.CommandLine = cmdLine.ToString();
Log.Debug(String.Format(
"Command line for pack {0}.{1} is '{2}' (task {3}). Cmdline object from PB is {4}",
engine.CompiledMode.ModeQName.PackageName ?? "''", engine.CompiledMode.ModeQName.ModeName ?? "''",
incarnation.CommandLine ?? "", engineState._taskDescription.TaskId,
cmdLine == null ? "null" : "not null"
));
if (String.IsNullOrWhiteSpace(incarnation.CommandLine))
{
throw new PackageBaseException("CommandLine generated by PackageBase is empty");
}
if (!engine.IsReadyToRun())
{
Log.Warn("PackageBase: task is not ready to run after PrepareInputs");
Log.Error("PackageBase: task is not ready to run after PrepareInputs");
throw new PackageBaseException("PackageBase: task is not ready to run after processing inputs");
}
return(incarnation);
}
public static IncarnationParams ProcessInputFiles(/* ref */ PackageEngineState engineState, out TimeSpan inputFilesTime)
{
var engine = new PackageEngine(engineState.CompiledDef, engineState.EngineContext);
var incarnation = new IncarnationParams();
var fileManager = new InputFileManager(engineState.StoragePathBase);
inputFilesTime = TimeSpanExt.Measure(() =>
{
TimeSpan timeLimit = PB_PROCESS_TIME_LIMIT;
try
{
Tools.ProcessWithTimeLimit(timeLimit, () =>
{
engine.PrepareInputs(fileManager);
CheckEngineResult(engine);
});
}
catch (TimeLimitException tle)
{
throw new PackageBaseException("Inputs processing in Package base exceeded time limit (" + timeLimit.TotalSeconds + " seconds).", tle);
}
finally
{
incarnation.FilesToCopy = incarnation.FilesToCopy.Concat(fileManager.GetInputFiles()).ToArray();
fileManager.Cleanup();
}
});
object cmdLine = engine.Ctx.Memory[CmdLineDef.DefName].Value;
//if (cmdLine == null)
//throw new PackageBaseException("CommandLine generated by PackageBase is empty");
incarnation.CommandLine = cmdLine.ToString();
Log.Debug(String.Format(
"Command line for pack {0}.{1} is '{2}' (task {3}). Cmdline object from PB is {4}",
engine.CompiledMode.ModeQName.PackageName ?? "''", engine.CompiledMode.ModeQName.ModeName ?? "''",
incarnation.CommandLine ?? "", engineState._taskDescription.TaskId,
cmdLine == null ? "null" : "not null"
));
if (String.IsNullOrWhiteSpace(incarnation.CommandLine))
throw new PackageBaseException("CommandLine generated by PackageBase is empty");
if (!engine.IsReadyToRun())
{
Log.Warn("PackageBase: task is not ready to run after PrepareInputs");
Log.Error("PackageBase: task is not ready to run after PrepareInputs");
throw new PackageBaseException("PackageBase: task is not ready to run after processing inputs");
}
return incarnation;
}
public HistorySample(
string package, string resourceName, NodeConfig[] nodesConfig,
Dictionary<string, string> packParams, Dictionary<string, double> modelCoefs,
TimeSpan calcTime, PackageEngine estimatorEngine)
{
Package = package;
ResourceName = resourceName;
NodesConfig = nodesConfig;
PackParams = packParams;
ModelCoefs = modelCoefs;
CalcTime = calcTime;
EstimatorEngine = estimatorEngine;
}
public HistorySample(
string package, string resourceName, NodeConfig[] nodesConfig,
Dictionary <string, string> packParams, Dictionary <string, double> modelCoefs,
TimeSpan calcTime, PackageEngine estimatorEngine)
{
Package = package;
ResourceName = resourceName;
NodesConfig = nodesConfig;
PackParams = packParams;
ModelCoefs = modelCoefs;
CalcTime = calcTime;
EstimatorEngine = estimatorEngine;
}
public static IDictionary <string, string> UpdateInputs(
/* ref */ PackageEngineState engineState,
IDictionary <string, string> moreParams)
{
var engine = new PackageEngine(engineState.CompiledDef, engineState.EngineContext);
engine.UpdateContractParams(moreParams);
var processedInputs = engine.GetEnabledInsValues().Where(inp => inp.IsSet);
var newInputParams = engine.Stringify(processedInputs, engine.CompiledMode.Defs.Values);
//var newInputParams = processedInputs.ToDictionary(
// inp => inp.VarName,
// inp => (inp.Value == null)? null: inp.Value.ToString()
//);
return(newInputParams);
}
public void Init(TaskDescription taskDescription, string storagePathBase)
{
StoragePathBase = storagePathBase;
if (_taskDescription == null)
{
_taskDescription = new TaskDescription(taskDescription);
}
if (CompiledDef == null)
{
CompiledDef = PackageBaseProxy.GetCompiledDef(taskDescription.Package);
}
if (EngineContext == null)
{
var engine = new PackageEngine(CompiledDef);
EngineContext = engine.Ctx;
}
}
public static Dictionary<NodeConfig, Estimation> GetEstimationsByModel(PackageEngineState engineState, IEnumerable<Resource> resources, IEnumerable<ResourceNode> permittedNodes)
{
var estims = new Dictionary<NodeConfig, Estimation>();
Log.Debug("Estimating by model...");
var engine = new PackageEngine(engineState.CompiledDef, engineState.EngineContext);
if (!engine.CanEstimate())
{
Log.Debug("Can't estimate by model");
return estims;
}
try
{
var allRes = resources.Select(r => new Common.Resource()
{
Name = r.ResourceName,
Nodes = r.Nodes.Select(n => new Common.Node()
{
ResourceName = n.ResourceName,
DNSName = n.NodeName,
Parameters = new Dictionary<string, string>(n.StaticHardwareParams),
CoresAvailable = n.CoresAvailable,
CoresTotal = (int)n.CoresCount,
}).ToArray(),
Parameters = new Dictionary<string, string>(r.HardwareParams),
});
Log.Debug("Permitted nodes: " + String.Join(", ", permittedNodes.Select(n => n.ResourceName + "." + n.NodeName)));
foreach (var node in permittedNodes)
{
try
{
var res = allRes.Single(r => r.Name == node.ResourceName);
var dest = new Common.LaunchDestination()
{
ResourceName = node.ResourceName,
NodeNames = new[] { node.NodeName },
};
var modelExecParams = new Dictionary<string, object>();
foreach (string paramId in TimeMeter.ClusterParameterReader.GetAvailableParameterIds()) // from scheduler. Why?
{
try { modelExecParams[paramId] = TimeMeter.ClusterParameterReader.GetValue(paramId, res, dest); }
catch (Exception) { /* it's ok not to extract all possible params */ }
}
var modelCoefs = GetModelCoefs(engine, node);
var modelEstimation = engine.Estimate(modelExecParams, modelCoefs);
if (engine.Ctx.Result.Messages.Any())
{
Log.Warn("Messages on estimation: " + String.Join("\n", engine.Ctx.Result.Messages.Select(mess => mess.Message)));
}
//double estimationInSeconds = engine.Estimate(modelExecParams, modelCoefs);
if (modelEstimation != null /* == no errors */ && modelEstimation.CalculationTime.IsSet &&
!Double.IsInfinity(modelEstimation.CalculationTime.Value) && !Double.IsNaN(modelEstimation.CalculationTime.Value))
{
var modelCoeffsToRemember = new Dictionary<string, double>();
foreach (var pair in modelCoefs)
{
if (pair.Value is double)
modelCoeffsToRemember[pair.Key] = (double)pair.Value;
}
estims.Add
(
new NodeConfig()
{
ResourceName = node.ResourceName,
NodeName = node.NodeName,
Cores = (uint) 1, // was node.CoresAvailable
// todo : node.pack.MinCores or node.pack.MaxCores
// todo : estimation from model -> cores
},
new Estimation(modelEstimation)
{
ModelCoeffs = modelCoeffsToRemember
}
);
/*
Log.Debug(String.Format("Estim by model on {0}.{1}: {2}",
node.ResourceName, node.NodeName,
(modelEstimation.CalculationTime.IsSet ? modelEstimation.CalculationTime.Value.ToString() : "not set")
));
*/
}
else
{
// todo : else Log.Trace estimation by model is NaN or Infinity
Log.Warn("Model estimation failed for task " + engineState._taskDescription.TaskId.ToString());
}
}
catch (Exception estimEx)
{
Log.Warn(String.Format(
"Exception while estimating task {1} on node '{2}' by models in PackageBase : {0}",
estimEx, engineState._taskDescription.TaskId, node.NodeName
));
}
}
}
catch (Exception e)
{
Log.Error(String.Format(
"Exception while getting estimations from models in PackageBase for task {2}: {0}\n{1}",
e.Message, e.StackTrace,
engineState._taskDescription.TaskId
));
}
return estims;
}
private static Dictionary<string, double> AutoAdjustCoefsByHistory(PackageEngine engine, ResourceNode node, Dictionary<string, object> fixedCoefs, Dictionary<string, double> adjustableCoefs)
{
lock (_historyLock)
{
//Log.Info("Adjusting coefs");
string packageName = engine.CompiledMode.ModeQName.PackageName.ToLowerInvariant();
var history = GetHistorySamples(packageName, node.ResourceName, node.NodeName);
if (history.Length < 1)
{
//Log.Debug("Not enough history samples to adjust coefs");
return null;
}
if (packageName == "bsm")
{
double[] fTime = history.Select(hs => double.Parse(hs.PackParams["ForecastSize"])).ToArray();
double[] cTime = history.Select(hs => hs.CalcTime.TotalSeconds).ToArray();
double perf = BSMSimpleEstimator.BsmEstimatePerf(fTime, cTime);
double dev = BSMSimpleEstimator.BsmRelativeStDev(perf, fTime, cTime);
return new Dictionary<string, double>()
{
{ "Perf", perf },
{ "D", dev }
};
}
else
{
Log.Warn("Unsupported package to adjust coefs: " + packageName);
return null;
}
}
}
private static void CheckEngineResult(PackageEngine engine)
{
CheckEngineResult(engine.Ctx.Result);
}
public static IDictionary<string, string> UpdateInputs(
/* ref */ PackageEngineState engineState,
IDictionary<string, string> moreParams)
{
var engine = new PackageEngine(engineState.CompiledDef, engineState.EngineContext);
engine.UpdateContractParams(moreParams);
var processedInputs = engine.GetEnabledInsValues().Where(inp => inp.IsSet);
var newInputParams = engine.Stringify(processedInputs, engine.CompiledMode.Defs.Values);
//var newInputParams = processedInputs.ToDictionary(
// inp => inp.VarName,
// inp => (inp.Value == null)? null: inp.Value.ToString()
//);
return newInputParams;
}
public static Dictionary<string, string> ProcessOutputs(PackageEngineState engineState, string ftpRoot, out IEnumerable<TaskFileDescription> outFiles, out TimeSpan outputFilesTime)
{
var forbiddenFileNamesRx = (new string[] {
@"\.\./", @"torque", @"\.dataList\.src$", @"\.hosts$", @"\.sh$", @"\.exe$"
}).Select(st => new Regex(st, RegexOptions.IgnoreCase));
string ftpRootWithoutSlash = ftpRoot.TrimEnd(new[] { '/', '\\' });
var allFilePaths = IOProxy.Ftp.GetFileNamesInAllTree(ftpRootWithoutSlash);
var filteredFilePathsWithRoot = allFilePaths.Where(st => !forbiddenFileNamesRx.Any(rx => rx.IsMatch(st)));
var filteredFilePaths = filteredFilePathsWithRoot.Select(st => st.Substring(ftpRoot.Length));
var fileManager = new OutputFileManager(ftpRoot, engineState.StoragePathBase);
var engine = new PackageEngine(engineState.CompiledDef, engineState.EngineContext);
var actionStarted = DateTime.Now;
{
TimeSpan timeLimit = PB_PROCESS_TIME_LIMIT;
try
{
Tools.ProcessWithTimeLimit(timeLimit, () =>
{
//engine.ProcessOutputs(outputFileNames, fileManager);
engine.ProcessOutputs(filteredFilePaths, fileManager);
});
}
catch (TimeLimitException tle)
{
throw new PackageBaseException("Outputs processing in Package base exceeded time limit (" + timeLimit.TotalSeconds + " seconds).", tle);
}
finally
{
outFiles = fileManager.GetOutputFiles(); // includes Cleanup();
}
}
var actionFinished = DateTime.Now;
outputFilesTime = actionFinished - actionStarted; // todo : fix ugliness
var processedOuts = engine.GetEnabledOutsValues();
var outFilesAndGroups = engine.GetEnabledOutDefs().Where(def => def is IOutFileDef || def is OutFileGroupDefBase);
foreach (var def in outFilesAndGroups)
{
var memoryEntry = processedOuts.First(me => me.VarName == def.Name);
if (memoryEntry.IsSet) // file was there
{
if (def is IOutFileDef)
{
var extFileDef = memoryEntry.Value as ExternalFileDef;
extFileDef.Locator = outFiles.FirstOrDefault(fd => fd.SlotName == def.Name).StorageId;
if (extFileDef.Locator == null) // todo : null.StorageId will fail earlier
{
Log.Warn("No output file processed for param " + memoryEntry.VarName + ", and memoryEntry.IsSet == true");
}
}
else
if (def is OutFileGroupDefBase)
{
var group = def as OutFileGroupDefBase;
var extFileDefs = memoryEntry.Value as IEnumerable<ExternalFileDef>;
foreach (var extFileDef in extFileDefs)
{
extFileDef.Locator = outFiles.FirstOrDefault(file =>
file.SlotName == def.Name &&
file.FileName == extFileDef.FileName
).StorageId;
if (extFileDef.Locator == null) // todo : null.StorageId will fail earlier
{
Log.Warn(String.Format(
"File '{0}' wasn't processed for param {1}, and memoryEntry.IsSet == true",
extFileDef.FileName, memoryEntry.VarName
));
}
}
}
}
}
//var outputParams = engine.Stringify(processedOuts, engine.CompiledMode.Outs.Values);
var outputParams = PackageBase.Serializer.PackageBaseSerializer.Serialize(processedOuts, engine.CompiledMode.Outs.Values);
Log.Debug(String.Format("Task's {0} output params are: {1}", engineState._taskDescription.TaskId,
String.Join(", ", outputParams.Select(pair => pair.Key + " = " + pair.Value))
));
return outputParams;
}
private static Dictionary<string, object> GetModelCoefs(PackageEngine engine, ResourceNode node)
{
var fixedCoefs = new Dictionary<string, object>(engine.CompiledMode.Models.DefaultCoeffs);
var adjustableCoefs = new Dictionary<string, double>();
string packageName = engine.CompiledMode.ModeQName.PackageName; // engineState._taskDescription.Package
var packParams = node.PackageByName(packageName).Params;
foreach (string paramKey in packParams.Keys)
{
string paramValue = packParams[paramKey];
double paramValueAsDouble;
bool isDouble = double.TryParse(paramValue,
System.Globalization.NumberStyles.Float,
System.Globalization.CultureInfo.InvariantCulture.NumberFormat,
out paramValueAsDouble);
if (paramKey.ToLowerInvariant().StartsWith(FIXED_COEF_PREFIX))
{
string coefName = paramKey.Remove(0, FIXED_COEF_PREFIX.Length);
if (isDouble)
fixedCoefs[coefName] = paramValueAsDouble;
else
fixedCoefs[coefName] = paramValue;
}
else
if (paramKey.ToLowerInvariant().StartsWith(ADJUSTABLE_COEF_PREFIX))
{
string coefName = paramKey.Remove(0, ADJUSTABLE_COEF_PREFIX.Length);
if (isDouble)
adjustableCoefs[coefName] = paramValueAsDouble;
else
{
fixedCoefs[coefName] = paramValue;
Log.Warn(String.Format(
"Cannot adjust param '{0}' for pack '{1}' on resource node '{2}.{3}' because it's value is not number",
paramKey, packageName,
node.ResourceName, node.NodeName
));
}
}
}
var adjustedCoefs = AutoAdjustCoefsByHistory(engine, node, fixedCoefs, adjustableCoefs) ?? new Dictionary<string, double>();
if (adjustableCoefs.Any())
Log.Info("Model coefs were adjusted");
var newCoefNames = adjustedCoefs.Keys.Except(adjustableCoefs.Keys);
if (newCoefNames.Any())
Log.Warn("Autoadjust created new coefs (ignoring them): " + String.Join(", ", newCoefNames));
var modelCoefs = new Dictionary<string, object>(fixedCoefs);
foreach (var coefName in adjustableCoefs.Keys)
{
if (adjustedCoefs.ContainsKey(coefName))
modelCoefs[coefName] = adjustedCoefs[coefName];
else
{
modelCoefs[coefName] = adjustableCoefs[coefName];
Log.Warn("Coef " + coefName + " was not returned as adjusted. Using non-adjusted value.");
}
}
return modelCoefs;
}
private static void CheckEngineResult(PackageEngine engine)
{
CheckEngineResult(engine.Ctx.Result);
}
public static Dictionary <string, string> ProcessOutputs(PackageEngineState engineState, string ftpRoot, out IEnumerable <TaskFileDescription> outFiles, out TimeSpan outputFilesTime)
{
var forbiddenFileNamesRx = (new string[] {
@"\.\./", @"torque", @"\.dataList\.src$", @"\.hosts$", @"\.sh$", @"\.exe$"
}).Select(st => new Regex(st, RegexOptions.IgnoreCase));
string ftpRootWithoutSlash = ftpRoot.TrimEnd(new[] { '/', '\\' });
var allFilePaths = IOProxy.Ftp.GetFileNamesInAllTree(ftpRootWithoutSlash);
var filteredFilePathsWithRoot = allFilePaths.Where(st => !forbiddenFileNamesRx.Any(rx => rx.IsMatch(st)));
var filteredFilePaths = filteredFilePathsWithRoot.Select(st => st.Substring(ftpRoot.Length));
var fileManager = new OutputFileManager(ftpRoot, engineState.StoragePathBase);
var engine = new PackageEngine(engineState.CompiledDef, engineState.EngineContext);
var actionStarted = DateTime.Now;
{
TimeSpan timeLimit = PB_PROCESS_TIME_LIMIT;
try
{
Tools.ProcessWithTimeLimit(timeLimit, () =>
{
//engine.ProcessOutputs(outputFileNames, fileManager);
engine.ProcessOutputs(filteredFilePaths, fileManager);
});
}
catch (TimeLimitException tle)
{
throw new PackageBaseException("Outputs processing in Package base exceeded time limit (" + timeLimit.TotalSeconds + " seconds).", tle);
}
finally
{
outFiles = fileManager.GetOutputFiles(); // includes Cleanup();
}
}
var actionFinished = DateTime.Now;
outputFilesTime = actionFinished - actionStarted; // todo : fix ugliness
var processedOuts = engine.GetEnabledOutsValues();
var outFilesAndGroups = engine.GetEnabledOutDefs().Where(def => def is IOutFileDef || def is OutFileGroupDefBase);
foreach (var def in outFilesAndGroups)
{
var memoryEntry = processedOuts.First(me => me.VarName == def.Name);
if (memoryEntry.IsSet) // file was there
{
if (def is IOutFileDef)
{
var extFileDef = memoryEntry.Value as ExternalFileDef;
extFileDef.Locator = outFiles.FirstOrDefault(fd => fd.SlotName == def.Name).StorageId;
if (extFileDef.Locator == null) // todo : null.StorageId will fail earlier
{
Log.Warn("No output file processed for param " + memoryEntry.VarName + ", and memoryEntry.IsSet == true");
}
}
else
if (def is OutFileGroupDefBase)
{
var group = def as OutFileGroupDefBase;
var extFileDefs = memoryEntry.Value as IEnumerable <ExternalFileDef>;
foreach (var extFileDef in extFileDefs)
{
extFileDef.Locator = outFiles.FirstOrDefault(file =>
file.SlotName == def.Name &&
file.FileName == extFileDef.FileName
).StorageId;
if (extFileDef.Locator == null) // todo : null.StorageId will fail earlier
{
Log.Warn(String.Format(
"File '{0}' wasn't processed for param {1}, and memoryEntry.IsSet == true",
extFileDef.FileName, memoryEntry.VarName
));
}
}
}
}
}
//var outputParams = engine.Stringify(processedOuts, engine.CompiledMode.Outs.Values);
var outputParams = PackageBase.Serializer.PackageBaseSerializer.Serialize(processedOuts, engine.CompiledMode.Outs.Values);
Log.Debug(String.Format("Task's {0} output params are: {1}", engineState._taskDescription.TaskId,
String.Join(", ", outputParams.Select(pair => pair.Key + " = " + pair.Value))
));
return(outputParams);
}
public static Dictionary <NodeConfig, Estimation> GetEstimationsByModel(PackageEngineState engineState, IEnumerable <Resource> resources, IEnumerable <ResourceNode> permittedNodes)
{
var estims = new Dictionary <NodeConfig, Estimation>();
Log.Debug("Estimating by model...");
var engine = new PackageEngine(engineState.CompiledDef, engineState.EngineContext);
if (!engine.CanEstimate())
{
Log.Debug("Can't estimate by model");
return(estims);
}
try
{
var allRes = resources.Select(r => new Common.Resource()
{
Name = r.ResourceName,
Nodes = r.Nodes.Select(n => new Common.Node()
{
ResourceName = n.ResourceName,
DNSName = n.NodeName,
Parameters = new Dictionary <string, string>(n.StaticHardwareParams),
CoresAvailable = n.CoresAvailable,
CoresTotal = (int)n.CoresCount,
}).ToArray(),
Parameters = new Dictionary <string, string>(r.HardwareParams),
});
Log.Debug("Permitted nodes: " + String.Join(", ", permittedNodes.Select(n => n.ResourceName + "." + n.NodeName)));
foreach (var node in permittedNodes)
{
try
{
var res = allRes.Single(r => r.Name == node.ResourceName);
var dest = new Common.LaunchDestination()
{
ResourceName = node.ResourceName,
NodeNames = new[] { node.NodeName },
};
var modelExecParams = new Dictionary <string, object>();
foreach (string paramId in TimeMeter.ClusterParameterReader.GetAvailableParameterIds()) // from scheduler. Why?
{
try { modelExecParams[paramId] = TimeMeter.ClusterParameterReader.GetValue(paramId, res, dest); }
catch (Exception) { /* it's ok not to extract all possible params */ }
}
var modelCoefs = GetModelCoefs(engine, node);
var modelEstimation = engine.Estimate(modelExecParams, modelCoefs);
if (engine.Ctx.Result.Messages.Any())
{
Log.Warn("Messages on estimation: " + String.Join("\n", engine.Ctx.Result.Messages.Select(mess => mess.Message)));
}
//double estimationInSeconds = engine.Estimate(modelExecParams, modelCoefs);
if (modelEstimation != null /* == no errors */ && modelEstimation.CalculationTime.IsSet &&
!Double.IsInfinity(modelEstimation.CalculationTime.Value) && !Double.IsNaN(modelEstimation.CalculationTime.Value))
{
var modelCoeffsToRemember = new Dictionary <string, double>();
foreach (var pair in modelCoefs)
{
if (pair.Value is double)
{
modelCoeffsToRemember[pair.Key] = (double)pair.Value;
}
}
estims.Add
(
new NodeConfig()
{
ResourceName = node.ResourceName,
NodeName = node.NodeName,
Cores = (uint)1, // was node.CoresAvailable
// todo : node.pack.MinCores or node.pack.MaxCores
// todo : estimation from model -> cores
},
new Estimation(modelEstimation)
{
ModelCoeffs = modelCoeffsToRemember
}
);
/*
* Log.Debug(String.Format("Estim by model on {0}.{1}: {2}",
* node.ResourceName, node.NodeName,
* (modelEstimation.CalculationTime.IsSet ? modelEstimation.CalculationTime.Value.ToString() : "not set")
* ));
*/
}
else
{
// todo : else Log.Trace estimation by model is NaN or Infinity
Log.Warn("Model estimation failed for task " + engineState._taskDescription.TaskId.ToString());
}
}
catch (Exception estimEx)
{
Log.Warn(String.Format(
"Exception while estimating task {1} on node '{2}' by models in PackageBase : {0}",
estimEx, engineState._taskDescription.TaskId, node.NodeName
));
}
}
}
catch (Exception e)
{
Log.Error(String.Format(
"Exception while getting estimations from models in PackageBase for task {2}: {0}\n{1}",
e.Message, e.StackTrace,
engineState._taskDescription.TaskId
));
}
return(estims);
}
private static Dictionary <string, object> GetModelCoefs(PackageEngine engine, ResourceNode node)
{
var fixedCoefs = new Dictionary <string, object>(engine.CompiledMode.Models.DefaultCoeffs);
var adjustableCoefs = new Dictionary <string, double>();
string packageName = engine.CompiledMode.ModeQName.PackageName; // engineState._taskDescription.Package
var packParams = node.PackageByName(packageName).Params;
foreach (string paramKey in packParams.Keys)
{
string paramValue = packParams[paramKey];
double paramValueAsDouble;
bool isDouble = double.TryParse(paramValue,
System.Globalization.NumberStyles.Float,
System.Globalization.CultureInfo.InvariantCulture.NumberFormat,
out paramValueAsDouble);
if (paramKey.ToLowerInvariant().StartsWith(FIXED_COEF_PREFIX))
{
string coefName = paramKey.Remove(0, FIXED_COEF_PREFIX.Length);
if (isDouble)
{
fixedCoefs[coefName] = paramValueAsDouble;
}
else
{
fixedCoefs[coefName] = paramValue;
}
}
else
if (paramKey.ToLowerInvariant().StartsWith(ADJUSTABLE_COEF_PREFIX))
{
string coefName = paramKey.Remove(0, ADJUSTABLE_COEF_PREFIX.Length);
if (isDouble)
{
adjustableCoefs[coefName] = paramValueAsDouble;
}
else
{
fixedCoefs[coefName] = paramValue;
Log.Warn(String.Format(
"Cannot adjust param '{0}' for pack '{1}' on resource node '{2}.{3}' because it's value is not number",
paramKey, packageName,
node.ResourceName, node.NodeName
));
}
}
}
var adjustedCoefs = AutoAdjustCoefsByHistory(engine, node, fixedCoefs, adjustableCoefs) ?? new Dictionary <string, double>();
if (adjustableCoefs.Any())
{
Log.Info("Model coefs were adjusted");
}
var newCoefNames = adjustedCoefs.Keys.Except(adjustableCoefs.Keys);
if (newCoefNames.Any())
{
Log.Warn("Autoadjust created new coefs (ignoring them): " + String.Join(", ", newCoefNames));
}
var modelCoefs = new Dictionary <string, object>(fixedCoefs);
foreach (var coefName in adjustableCoefs.Keys)
{
if (adjustedCoefs.ContainsKey(coefName))
{
modelCoefs[coefName] = adjustedCoefs[coefName];
}
else
{
modelCoefs[coefName] = adjustableCoefs[coefName];
Log.Warn("Coef " + coefName + " was not returned as adjusted. Using non-adjusted value.");
}
}
return(modelCoefs);
}
public void Init(TaskDescription taskDescription, string storagePathBase)
{
StoragePathBase = storagePathBase;
if (_taskDescription == null)
_taskDescription = new TaskDescription(taskDescription);
if (CompiledDef == null)
CompiledDef = PackageBaseProxy.GetCompiledDef(taskDescription.Package);
if (EngineContext == null)
{
var engine = new PackageEngine(CompiledDef);
EngineContext = engine.Ctx;
}
}
