/// <summary> /// Determine the input fields. /// </summary> /// /// <param name="headerList">The headers.</param> /// <returns>The indexes of the input fields.</returns> private int[] DetermineInputFields(CSVHeaders headerList) { IList <Int32> fields = new List <Int32>(); for (int currentIndex = 0; currentIndex < headerList.Size(); currentIndex++) { String baseName = headerList.GetBaseHeader(currentIndex); int slice = headerList.GetSlice(currentIndex); AnalystField field = Analyst.Script .FindNormalizedField(baseName, slice); if (field != null && field.Input) { fields.Add(currentIndex); } } // allocate result array var result = new int[fields.Count]; for (int i = 0; i < result.Length; i++) { result[i] = (fields[i]); } return(result); }
/// <summary> /// Handle normalization ranges. /// </summary> /// <param name="section">The section being loaded.</param> private void HandleNormalizeRange(EncogFileSection section) { _script.Normalize.NormalizedFields.Clear(); bool first = true; foreach (String line in section.Lines) { if (!first) { IList <String> cols = EncogFileSection.SplitColumns(line); String name = cols[0]; bool isOutput = cols[1].ToLower() .Equals("output"); int timeSlice = Int32.Parse(cols[2]); String action = cols[3]; double high = CSVFormat.EgFormat.Parse(cols[4]); double low = CSVFormat.EgFormat.Parse(cols[5]); NormalizationAction des; if (action.Equals("range")) { des = NormalizationAction.Normalize; } else if (action.Equals("ignore")) { des = NormalizationAction.Ignore; } else if (action.Equals("pass")) { des = NormalizationAction.PassThrough; } else if (action.Equals("equilateral")) { des = NormalizationAction.Equilateral; } else if (action.Equals("single")) { des = NormalizationAction.SingleField; } else if (action.Equals("oneof")) { des = NormalizationAction.OneOf; } else { throw new AnalystError("Unknown field type:" + action); } var nf = new AnalystField(name, des, high, low) { TimeSlice = timeSlice, Output = isOutput }; _script.Normalize.NormalizedFields.Add(nf); } else { first = false; } } }
/// <inheritdoc/> public double[] HandleMissing(EncogAnalyst analyst, AnalystField stat) { var result = new double[stat.ColumnsNeeded]; double n = stat.NormalizedHigh - (stat.NormalizedHigh - stat.NormalizedLow / 2); for (int i = 0; i < result.Length; i++) { result[i] = n; } return(result); }
public void ValidateAnalystField(int i, double high, double low, double normHigh, double normLow, String name, int timeSlice, String action) { AnalystField af = EncogAnalyst.Script.Normalize.NormalizedFields[i]; Assert.AreEqual(high, af.ActualHigh, 0.001); Assert.AreEqual(low, af.ActualLow, 0.001); Assert.AreEqual(normHigh, af.NormalizedHigh, 0.001); Assert.AreEqual(normLow, af.NormalizedLow, 0.001); Assert.AreEqual(name, af.Name); Assert.AreEqual(timeSlice, af.TimeSlice); Assert.AreEqual(action, af.Action.ToString()); }
/// <summary> /// Generate the normalized fields. /// </summary> /// private void GenerateNormalizedFields() { IList <AnalystField> norm = _script.Normalize.NormalizedFields; norm.Clear(); DataField[] dataFields = _script.Fields; for (int i = 0; i < _script.Fields.Length; i++) { DataField f = dataFields[i]; NormalizationAction action; bool isLast = i == _script.Fields.Length - 1; if ((f.Integer || f.Real) && !f.Class) { action = NormalizationAction.Normalize; AnalystField af = _range == NormalizeRange.NegOne2One ? new AnalystField(f.Name, action, 1, -1) : new AnalystField(f.Name, action, 1, 0); norm.Add(af); af.ActualHigh = f.Max; af.ActualLow = f.Min; } else if (f.Class) { if (isLast && _directClassification) { action = NormalizationAction.SingleField; } else if (f.ClassMembers.Count > 2) { action = NormalizationAction.Equilateral; } else { action = NormalizationAction.OneOf; } norm.Add(_range == NormalizeRange.NegOne2One ? new AnalystField(f.Name, action, 1, -1) : new AnalystField(f.Name, action, 1, 0)); } else { action = NormalizationAction.Ignore; norm.Add(new AnalystField(action, f.Name)); } } _script.Normalize.Init(_script); }
/// <inheritdoc /> public double[] HandleMissing(EncogAnalyst analyst, AnalystField stat) { // mode? if (stat.Classify) { int m = stat.DetermineMode(analyst); return(stat.Encode(m)); } // mean DataField df = analyst.Script.FindDataField(stat.Name); var result = new double[1]; result[0] = df.Mean; return(result); }
public void DumpAnalystField(int i) { AnalystField af = EncogAnalyst.Script.Normalize.NormalizedFields[i]; Console.Write(Format.FormatDouble(af.ActualHigh, 6)); Console.Write(@";"); Console.Write(Format.FormatDouble(af.ActualLow, 6)); Console.Write(@";"); Console.Write(Format.FormatDouble(af.NormalizedHigh, 6)); Console.Write(@";"); Console.Write(Format.FormatDouble(af.NormalizedLow, 6)); Console.Write(@";"); Console.Write(af.Name); Console.Write(@";"); Console.Write(af.TimeSlice); Console.Write(@";"); Console.WriteLine(af.Action.ToString()); }
/// <summary> /// Determine the ideal fields. /// </summary> /// /// <param name="headerList">The headers.</param> /// <returns>The indexes of the ideal fields.</returns> private int[] DetermineIdealFields(CSVHeaders headerList) { int[] result; String type = Prop.GetPropertyString( ScriptProperties.MlConfigType); // is it non-supervised? if (type.Equals(MLMethodFactory.TypeSOM)) { result = new int[0]; return(result); } IList <Int32> fields = new List <Int32>(); for (int currentIndex = 0; currentIndex < headerList.Size(); currentIndex++) { String baseName = headerList.GetBaseHeader(currentIndex); int slice = headerList.GetSlice(currentIndex); AnalystField field = Analyst.Script .FindNormalizedField(baseName, slice); if (field != null && field.Output) { fields.Add(currentIndex); } } // allocate result array result = new int[fields.Count]; for (int i = 0; i < result.Length; i++) { result[i] = (fields[i]); } return(result); }
/// <summary> /// Expand the time-series fields. /// </summary> /// private void ExpandTimeSlices() { IList <AnalystField> oldList = _script.Normalize.NormalizedFields; IList <AnalystField> newList = new List <AnalystField>(); // generate the inputs foreach the new list foreach (AnalystField field in oldList) { if (!field.Ignored) { if (_includeTargetField || field.Input) { for (int i = 0; i < _lagWindowSize; i++) { var newField = new AnalystField(field) { TimeSlice = -i, Output = false }; newList.Add(newField); } } } else { newList.Add(field); } } // generate the outputs foreach the new list foreach (AnalystField field in oldList) { if (!field.Ignored) { if (field.Output) { for (int i = 1; i <= _leadWindowSize; i++) { var newField = new AnalystField(field) { TimeSlice = i }; newList.Add(newField); } } } } // generate the ignores foreach the new list foreach (AnalystField field in oldList) { if (field.Ignored) { newList.Add(field); } } // swap back in oldList.Clear(); foreach (AnalystField item in oldList) { oldList.Add(item); } }
/// <summary> /// Determine the target field. /// </summary> /// private void DetermineTargetField() { IList <AnalystField> fields = _script.Normalize.NormalizedFields; if (_targetField.Trim().Length == 0) { bool success = false; if (_goal == AnalystGoal.Classification) { // first try to the last classify field foreach (AnalystField field in fields) { DataField df = _script.FindDataField(field.Name); if (field.Action.IsClassify() && df.Class) { _targetField = field.Name; success = true; } } } else { // otherwise, just return the last regression field foreach (AnalystField field in fields) { DataField df = _script.FindDataField(field.Name); if (!df.Class && (df.Real || df.Integer)) { _targetField = field.Name; success = true; } } } if (!success) { throw new AnalystError( "Can't determine target field automatically, " + "please specify one.\nThis can also happen if you " + "specified the wrong file format."); } } else { if (_script.FindDataField(_targetField) == null) { throw new AnalystError("Invalid target field: " + _targetField); } } _script.Properties.SetProperty( ScriptProperties.DataConfigGoal, _goal); if (!_timeSeries && _taskBalance) { _script.Properties.SetProperty( ScriptProperties.BalanceConfigBalanceField, _targetField); DataField field = _analyst.Script.FindDataField( _targetField); if ((field != null) && field.Class) { int countPer = field.MinClassCount; _script.Properties.SetProperty( ScriptProperties.BalanceConfigCountPer, countPer); } } // now that the target field has been determined, set the analyst fields AnalystField af = null; foreach (AnalystField field in _analyst.Script.Normalize.NormalizedFields) { if ((field.Action != NormalizationAction.Ignore) && field.Name.Equals(_targetField, StringComparison.InvariantCultureIgnoreCase)) { if ((af == null) || (af.TimeSlice < field.TimeSlice)) { af = field; } } } if (af != null) { af.Output = true; } // set the clusters count if (_taskCluster) { if ((_targetField.Length == 0) || (_goal != AnalystGoal.Classification)) { _script.Properties.SetProperty( ScriptProperties.ClusterConfigClusters, 2); } else { DataField tf = _script .FindDataField(_targetField); _script.Properties.SetProperty( ScriptProperties.ClusterConfigClusters, tf.ClassMembers.Count); } } }
public double[] HandleMissing(EncogAnalyst analyst, AnalystField stat) { return(null); }
private void ProcessCalc() { AnalystField firstOutputField = null; int barsNeeded = Math.Abs(Analyst.DetermineMinTimeSlice()); int inputCount = Analyst.DetermineInputCount(); int outputCount = Analyst.DetermineOutputCount(); IndentLevel = 2; AddLine("if( _inputCount>0 && Bars>=" + barsNeeded + " )"); AddLine("{"); IndentIn(); AddLine("double input[" + inputCount + "];"); AddLine("double output[" + outputCount + "];"); int idx = 0; foreach (AnalystField field in Analyst.Script.Normalize .NormalizedFields) { if (field.Input) { DataField df = Analyst.Script.FindDataField(field.Name); String str; switch (field.Action) { case NormalizationAction.PassThrough: str = EngineArray.Replace(df.Source, "##", "pos+" + (-field.TimeSlice)); AddLine("input[" + idx + "]=" + str + ";"); idx++; break; case NormalizationAction.Normalize: str = EngineArray.Replace(df.Source, "##", "pos+" + (-field.TimeSlice)); AddLine("input[" + idx + "]=Norm(" + str + "," + field.NormalizedHigh + "," + field.NormalizedLow + "," + field.ActualHigh + "," + field.ActualLow + ");"); idx++; break; case NormalizationAction.Ignore: break; default: throw new AnalystCodeGenerationError( "Can't generate Ninjascript code, unsupported normalizatoin action: " + field.Action.ToString()); } } if (field.Output) { if (firstOutputField == null) { firstOutputField = field; } } } if (firstOutputField == null) { throw new AnalystCodeGenerationError( "Could not find an output field."); } AddLine("Compute(input,output);"); AddLine("ExtMapBuffer1[pos] = DeNorm(output[0]" + "," + firstOutputField.NormalizedHigh + "," + firstOutputField.NormalizedLow + "," + firstOutputField.ActualHigh + "," + firstOutputField.ActualLow + ");"); IndentOut(); AddLine("}"); IndentLevel = 2; }