/// <summary>
/// Determine the input fields.
/// </summary>
///
/// <param name="headerList">The headers.</param>
/// <returns>The indexes of the input fields.</returns>
private int[] DetermineInputFields(CSVHeaders headerList)
{
IList <Int32> fields = new List <Int32>();
for (int currentIndex = 0; currentIndex < headerList.Size(); currentIndex++)
{
String baseName = headerList.GetBaseHeader(currentIndex);
int slice = headerList.GetSlice(currentIndex);
AnalystField field = Analyst.Script
.FindNormalizedField(baseName, slice);
if (field != null && field.Input)
{
fields.Add(currentIndex);
}
}
// allocate result array
var result = new int[fields.Count];
for (int i = 0; i < result.Length; i++)
{
result[i] = (fields[i]);
}
return(result);
}
/// <summary>
/// Handle normalization ranges.
/// </summary>
/// <param name="section">The section being loaded.</param>
private void HandleNormalizeRange(EncogFileSection section)
{
_script.Normalize.NormalizedFields.Clear();
bool first = true;
foreach (String line in section.Lines)
{
if (!first)
{
IList <String> cols = EncogFileSection.SplitColumns(line);
String name = cols[0];
bool isOutput = cols[1].ToLower()
.Equals("output");
int timeSlice = Int32.Parse(cols[2]);
String action = cols[3];
double high = CSVFormat.EgFormat.Parse(cols[4]);
double low = CSVFormat.EgFormat.Parse(cols[5]);
NormalizationAction des;
if (action.Equals("range"))
{
des = NormalizationAction.Normalize;
}
else if (action.Equals("ignore"))
{
des = NormalizationAction.Ignore;
}
else if (action.Equals("pass"))
{
des = NormalizationAction.PassThrough;
}
else if (action.Equals("equilateral"))
{
des = NormalizationAction.Equilateral;
}
else if (action.Equals("single"))
{
des = NormalizationAction.SingleField;
}
else if (action.Equals("oneof"))
{
des = NormalizationAction.OneOf;
}
else
{
throw new AnalystError("Unknown field type:" + action);
}
var nf = new AnalystField(name, des, high, low)
{
TimeSlice = timeSlice, Output = isOutput
};
_script.Normalize.NormalizedFields.Add(nf);
}
else
{
first = false;
}
}
}
/// <inheritdoc/>
public double[] HandleMissing(EncogAnalyst analyst, AnalystField stat)
{
var result = new double[stat.ColumnsNeeded];
double n = stat.NormalizedHigh - (stat.NormalizedHigh - stat.NormalizedLow / 2);
for (int i = 0; i < result.Length; i++)
{
result[i] = n;
}
return(result);
}
public void ValidateAnalystField(int i, double high, double low, double normHigh, double normLow,
String name, int timeSlice, String action)
{
AnalystField af = EncogAnalyst.Script.Normalize.NormalizedFields[i];
Assert.AreEqual(high, af.ActualHigh, 0.001);
Assert.AreEqual(low, af.ActualLow, 0.001);
Assert.AreEqual(normHigh, af.NormalizedHigh, 0.001);
Assert.AreEqual(normLow, af.NormalizedLow, 0.001);
Assert.AreEqual(name, af.Name);
Assert.AreEqual(timeSlice, af.TimeSlice);
Assert.AreEqual(action, af.Action.ToString());
}
/// <summary>
/// Generate the normalized fields.
/// </summary>
///
private void GenerateNormalizedFields()
{
IList <AnalystField> norm = _script.Normalize.NormalizedFields;
norm.Clear();
DataField[] dataFields = _script.Fields;
for (int i = 0; i < _script.Fields.Length; i++)
{
DataField f = dataFields[i];
NormalizationAction action;
bool isLast = i == _script.Fields.Length - 1;
if ((f.Integer || f.Real) && !f.Class)
{
action = NormalizationAction.Normalize;
AnalystField af = _range == NormalizeRange.NegOne2One ? new AnalystField(f.Name, action, 1, -1) : new AnalystField(f.Name, action, 1, 0);
norm.Add(af);
af.ActualHigh = f.Max;
af.ActualLow = f.Min;
}
else if (f.Class)
{
if (isLast && _directClassification)
{
action = NormalizationAction.SingleField;
}
else if (f.ClassMembers.Count > 2)
{
action = NormalizationAction.Equilateral;
}
else
{
action = NormalizationAction.OneOf;
}
norm.Add(_range == NormalizeRange.NegOne2One
? new AnalystField(f.Name, action, 1, -1)
: new AnalystField(f.Name, action, 1, 0));
}
else
{
action = NormalizationAction.Ignore;
norm.Add(new AnalystField(action, f.Name));
}
}
_script.Normalize.Init(_script);
}
/// <inheritdoc />
public double[] HandleMissing(EncogAnalyst analyst, AnalystField stat)
{
// mode?
if (stat.Classify)
{
int m = stat.DetermineMode(analyst);
return(stat.Encode(m));
}
// mean
DataField df = analyst.Script.FindDataField(stat.Name);
var result = new double[1];
result[0] = df.Mean;
return(result);
}
public void DumpAnalystField(int i)
{
AnalystField af = EncogAnalyst.Script.Normalize.NormalizedFields[i];
Console.Write(Format.FormatDouble(af.ActualHigh, 6));
Console.Write(@";");
Console.Write(Format.FormatDouble(af.ActualLow, 6));
Console.Write(@";");
Console.Write(Format.FormatDouble(af.NormalizedHigh, 6));
Console.Write(@";");
Console.Write(Format.FormatDouble(af.NormalizedLow, 6));
Console.Write(@";");
Console.Write(af.Name);
Console.Write(@";");
Console.Write(af.TimeSlice);
Console.Write(@";");
Console.WriteLine(af.Action.ToString());
}
/// <summary>
/// Determine the ideal fields.
/// </summary>
///
/// <param name="headerList">The headers.</param>
/// <returns>The indexes of the ideal fields.</returns>
private int[] DetermineIdealFields(CSVHeaders headerList)
{
int[] result;
String type = Prop.GetPropertyString(
ScriptProperties.MlConfigType);
// is it non-supervised?
if (type.Equals(MLMethodFactory.TypeSOM))
{
result = new int[0];
return(result);
}
IList <Int32> fields = new List <Int32>();
for (int currentIndex = 0; currentIndex < headerList.Size(); currentIndex++)
{
String baseName = headerList.GetBaseHeader(currentIndex);
int slice = headerList.GetSlice(currentIndex);
AnalystField field = Analyst.Script
.FindNormalizedField(baseName, slice);
if (field != null && field.Output)
{
fields.Add(currentIndex);
}
}
// allocate result array
result = new int[fields.Count];
for (int i = 0; i < result.Length; i++)
{
result[i] = (fields[i]);
}
return(result);
}
/// <summary>
/// Expand the time-series fields.
/// </summary>
///
private void ExpandTimeSlices()
{
IList <AnalystField> oldList = _script.Normalize.NormalizedFields;
IList <AnalystField> newList = new List <AnalystField>();
// generate the inputs foreach the new list
foreach (AnalystField field in oldList)
{
if (!field.Ignored)
{
if (_includeTargetField || field.Input)
{
for (int i = 0; i < _lagWindowSize; i++)
{
var newField = new AnalystField(field)
{
TimeSlice = -i, Output = false
};
newList.Add(newField);
}
}
}
else
{
newList.Add(field);
}
}
// generate the outputs foreach the new list
foreach (AnalystField field in oldList)
{
if (!field.Ignored)
{
if (field.Output)
{
for (int i = 1; i <= _leadWindowSize; i++)
{
var newField = new AnalystField(field)
{
TimeSlice = i
};
newList.Add(newField);
}
}
}
}
// generate the ignores foreach the new list
foreach (AnalystField field in oldList)
{
if (field.Ignored)
{
newList.Add(field);
}
}
// swap back in
oldList.Clear();
foreach (AnalystField item in oldList)
{
oldList.Add(item);
}
}
/// <summary>
/// Determine the target field.
/// </summary>
///
private void DetermineTargetField()
{
IList <AnalystField> fields = _script.Normalize.NormalizedFields;
if (_targetField.Trim().Length == 0)
{
bool success = false;
if (_goal == AnalystGoal.Classification)
{
// first try to the last classify field
foreach (AnalystField field in fields)
{
DataField df = _script.FindDataField(field.Name);
if (field.Action.IsClassify() && df.Class)
{
_targetField = field.Name;
success = true;
}
}
}
else
{
// otherwise, just return the last regression field
foreach (AnalystField field in fields)
{
DataField df = _script.FindDataField(field.Name);
if (!df.Class && (df.Real || df.Integer))
{
_targetField = field.Name;
success = true;
}
}
}
if (!success)
{
throw new AnalystError(
"Can't determine target field automatically, "
+ "please specify one.\nThis can also happen if you "
+ "specified the wrong file format.");
}
}
else
{
if (_script.FindDataField(_targetField) == null)
{
throw new AnalystError("Invalid target field: "
+ _targetField);
}
}
_script.Properties.SetProperty(
ScriptProperties.DataConfigGoal, _goal);
if (!_timeSeries && _taskBalance)
{
_script.Properties.SetProperty(
ScriptProperties.BalanceConfigBalanceField,
_targetField);
DataField field = _analyst.Script.FindDataField(
_targetField);
if ((field != null) && field.Class)
{
int countPer = field.MinClassCount;
_script.Properties.SetProperty(
ScriptProperties.BalanceConfigCountPer, countPer);
}
}
// now that the target field has been determined, set the analyst fields
AnalystField af = null;
foreach (AnalystField field in _analyst.Script.Normalize.NormalizedFields)
{
if ((field.Action != NormalizationAction.Ignore) &&
field.Name.Equals(_targetField, StringComparison.InvariantCultureIgnoreCase))
{
if ((af == null) || (af.TimeSlice < field.TimeSlice))
{
af = field;
}
}
}
if (af != null)
{
af.Output = true;
}
// set the clusters count
if (_taskCluster)
{
if ((_targetField.Length == 0) ||
(_goal != AnalystGoal.Classification))
{
_script.Properties.SetProperty(
ScriptProperties.ClusterConfigClusters, 2);
}
else
{
DataField tf = _script
.FindDataField(_targetField);
_script.Properties.SetProperty(
ScriptProperties.ClusterConfigClusters,
tf.ClassMembers.Count);
}
}
}
public double[] HandleMissing(EncogAnalyst analyst, AnalystField stat)
{
return(null);
}
private void ProcessCalc()
{
AnalystField firstOutputField = null;
int barsNeeded = Math.Abs(Analyst.DetermineMinTimeSlice());
int inputCount = Analyst.DetermineInputCount();
int outputCount = Analyst.DetermineOutputCount();
IndentLevel = 2;
AddLine("if( _inputCount>0 && Bars>=" + barsNeeded + " )");
AddLine("{");
IndentIn();
AddLine("double input[" + inputCount + "];");
AddLine("double output[" + outputCount + "];");
int idx = 0;
foreach (AnalystField field in Analyst.Script.Normalize
.NormalizedFields)
{
if (field.Input)
{
DataField df = Analyst.Script.FindDataField(field.Name);
String str;
switch (field.Action)
{
case NormalizationAction.PassThrough:
str = EngineArray.Replace(df.Source, "##", "pos+"
+ (-field.TimeSlice));
AddLine("input[" + idx + "]=" + str + ";");
idx++;
break;
case NormalizationAction.Normalize:
str = EngineArray.Replace(df.Source, "##", "pos+"
+ (-field.TimeSlice));
AddLine("input[" + idx + "]=Norm(" + str + ","
+ field.NormalizedHigh + ","
+ field.NormalizedLow + ","
+ field.ActualHigh + ","
+ field.ActualLow + ");");
idx++;
break;
case NormalizationAction.Ignore:
break;
default:
throw new AnalystCodeGenerationError(
"Can't generate Ninjascript code, unsupported normalizatoin action: "
+ field.Action.ToString());
}
}
if (field.Output)
{
if (firstOutputField == null)
{
firstOutputField = field;
}
}
}
if (firstOutputField == null)
{
throw new AnalystCodeGenerationError(
"Could not find an output field.");
}
AddLine("Compute(input,output);");
AddLine("ExtMapBuffer1[pos] = DeNorm(output[0]" + ","
+ firstOutputField.NormalizedHigh + ","
+ firstOutputField.NormalizedLow + ","
+ firstOutputField.ActualHigh + ","
+ firstOutputField.ActualLow + ");");
IndentOut();
AddLine("}");
IndentLevel = 2;
}
