public void Analyze(EncogAnalyst theAnalyst, FileInfo inputFile, bool headers, CSVFormat format)
{
base.InputFilename = inputFile;
if ((((uint) headers) & 0) != 0)
{
goto Label_0063;
}
Label_005C:
base.ExpectInputHeaders = headers;
Label_0063:
base.InputFormat = format;
base.Analyzed = true;
this._x554f16462d8d4675 = theAnalyst;
base.PerformBasicCounts();
if (((uint) headers) >= 0)
{
this._x146688677da5adf5 = base.InputHeadings.Length;
this._x1402a42b31a31090 = this._x554f16462d8d4675.DetermineOutputFieldCount();
this._xc5416b6511261016 = new CSVHeaders(base.InputHeadings);
this._x7acb8518c8ed6133 = new TimeSeriesUtil(this._x554f16462d8d4675, false, this._xc5416b6511261016.Headers);
if (0 != 0)
{
goto Label_005C;
}
}
}
/// <summary>
/// Analyze the data. This counts the records and prepares the data to be
/// processed.
/// </summary>
/// <param name="theAnalyst">The analyst to use.</param>
/// <param name="inputFile">The input file to analyze.</param>
/// <param name="headers">True, if the input file has headers.</param>
/// <param name="format">The format of the input file.</param>
public void Analyze(EncogAnalyst theAnalyst,
FileInfo inputFile, bool headers, CSVFormat format)
{
InputFilename = inputFile;
ExpectInputHeaders = headers;
Format = format;
Analyzed = true;
_analyst = theAnalyst;
_data = new BasicMLDataSet();
ResetStatus();
int recordCount = 0;
int outputLength = _analyst.DetermineTotalColumns();
var csv = new ReadCSV(InputFilename.ToString(),
ExpectInputHeaders, Format);
ReadHeaders(csv);
_analystHeaders = new CSVHeaders(InputHeadings);
while (csv.Next() && !ShouldStop())
{
UpdateStatus(true);
double[] inputArray = AnalystNormalizeCSV.ExtractFields(
_analyst, _analystHeaders, csv, outputLength, true);
IMLData input = new BasicMLData(inputArray);
_data.Add(new BasicMLDataPair(input));
recordCount++;
}
RecordCount = recordCount;
Count = csv.ColumnCount;
ReadHeaders(csv);
csv.Close();
ReportDone(true);
}
/// <summary>
/// Determine the ideal fields.
/// </summary>
///
/// <param name="headerList">The headers.</param>
/// <returns>The indexes of the ideal fields.</returns>
private int[] DetermineIdealFields(CSVHeaders headerList)
{
int[] result;
String type = Prop.GetPropertyString(
ScriptProperties.MlConfigType);
// is it non-supervised?
if (type.Equals(MLMethodFactory.TypeSOM))
{
result = new int[0];
return result;
}
IList<Int32> fields = new List<Int32>();
for (int currentIndex = 0; currentIndex < headerList.Size(); currentIndex++)
{
String baseName = headerList.GetBaseHeader(currentIndex);
int slice = headerList.GetSlice(currentIndex);
AnalystField field = Analyst.Script
.FindNormalizedField(baseName, slice);
if (field != null && field.Output)
{
fields.Add(currentIndex);
}
}
// allocate result array
result = new int[fields.Count];
for (int i = 0; i < result.Length; i++)
{
result[i] = (fields[i]);
}
return result;
}
public override sealed bool ExecuteCommand(string args)
{
FileInfo info;
FileInfo info2;
bool flag;
string propertyString = base.Prop.GetPropertyString("GENERATE:CONFIG_sourceFile");
string sourceID = base.Prop.GetPropertyString("GENERATE:CONFIG_targetFile");
CSVFormat format = base.Analyst.Script.DetermineInputFormat(propertyString);
EncogLogging.Log(0, "Beginning generate");
if (((uint) flag) <= uint.MaxValue)
{
EncogLogging.Log(0, "source file:" + propertyString);
EncogLogging.Log(0, "target file:" + sourceID);
info = base.Script.ResolveFilename(propertyString);
if (0 == 0)
{
do
{
info2 = base.Script.ResolveFilename(sourceID);
}
while (4 == 0);
}
else
{
goto Label_00F5;
}
}
base.Script.MarkGenerated(sourceID);
flag = base.Script.ExpectInputHeaders(propertyString);
CSVHeaders headers = new CSVHeaders(info, flag, format);
int[] input = this.x163e1f9de31a8b41(headers);
int[] ideal = this.x176a88b9713cb7be(headers);
EncogUtility.ConvertCSV2Binary(info, format, info2, input, ideal, flag);
Label_00F5:
return false;
}
/// <summary>
/// Analyze the data. This counts the records and prepares the data to be
/// processed.
/// </summary>
///
/// <param name="theAnalyst">The analyst to use.</param>
/// <param name="inputFile">The input file.</param>
/// <param name="headers">True if headers are present.</param>
/// <param name="format">The format.</param>
public void Analyze(EncogAnalyst theAnalyst,
FileInfo inputFile, bool headers, CSVFormat format)
{
InputFilename = inputFile;
ExpectInputHeaders = headers;
Format = format;
Analyzed = true;
_analyst = theAnalyst;
PerformBasicCounts();
_fileColumns = InputHeadings.Length;
_outputColumns = _analyst.DetermineOutputFieldCount();
_analystHeaders = new CSVHeaders(InputHeadings);
_series = new TimeSeriesUtil(_analyst, false,
_analystHeaders.Headers);
}
/// <summary>
/// </summary>
public override sealed bool ExecuteCommand(String args)
{
// get filenames
String sourceID = Prop.GetPropertyString(
ScriptProperties.GenerateConfigSourceFile);
String targetID = Prop.GetPropertyString(
ScriptProperties.GenerateConfigTargetFile);
CSVFormat format = Analyst.Script.DetermineFormat();
EncogLogging.Log(EncogLogging.LevelDebug, "Beginning generate");
EncogLogging.Log(EncogLogging.LevelDebug, "source file:" + sourceID);
EncogLogging.Log(EncogLogging.LevelDebug, "target file:" + targetID);
FileInfo sourceFile = Script.ResolveFilename(sourceID);
FileInfo targetFile = Script.ResolveFilename(targetID);
// mark generated
Script.MarkGenerated(targetID);
// read file
bool headers = Script.ExpectInputHeaders(sourceID);
var headerList = new CSVHeaders(sourceFile, headers,
format);
int[] input = DetermineInputFields(headerList);
int[] ideal = DetermineIdealFields(headerList);
EncogUtility.ConvertCSV2Binary(sourceFile, format, targetFile, input,
ideal, headers);
return false;
}
/// <summary>
/// Determine the input fields.
/// </summary>
/// <param name="headerList">The headers.</param>
/// <returns>The indexes of the input fields.</returns>
private int[] DetermineInputFields(CSVHeaders headerList)
{
IList<Int32> fields = new List<Int32>();
for (int currentIndex = 0; currentIndex < headerList.Size(); currentIndex++)
{
String baseName = headerList.GetBaseHeader(currentIndex);
int slice = headerList.GetSlice(currentIndex);
AnalystField field = Analyst.Script
.FindNormalizedField(baseName, slice);
if (field != null && field.Input)
{
fields.Add(currentIndex);
}
}
// allocate result array
var result = new int[fields.Count];
for (int i = 0; i < result.Length; i++)
{
result[i] = (fields[i]);
}
return result;
}
private void x7e640a0bc3eeb27e(ReadCSV xe4aa442e12986e06)
{
int num;
CSVHeaders headers = new CSVHeaders(xe4aa442e12986e06.ColumnNames);
if (3 != 0)
{
this._xa942970cc8a85fd4 = new AnalyzedField[xe4aa442e12986e06.ColumnCount];
num = 0;
goto Label_0031;
}
Label_0013:
this._xa942970cc8a85fd4[num] = new AnalyzedField(this._x594135906c55045c, headers.GetHeader(num));
num++;
Label_0031:
if (num >= this._xa942970cc8a85fd4.Length)
{
return;
}
if (num >= xe4aa442e12986e06.ColumnCount)
{
throw new AnalystError("CSV header count does not match column count");
}
if (((uint) num) >= 0)
{
}
goto Label_0013;
}
/// <summary>
/// Extract fields from a file into a numeric array for machine learning.
/// </summary>
///
/// <param name="analyst">The analyst to use.</param>
/// <param name="headers">The headers for the input data.</param>
/// <param name="csv">The CSV that holds the input data.</param>
/// <param name="outputLength">The length of the returned array.</param>
/// <param name="skipOutput">True if the output should be skipped.</param>
/// <returns>The encoded data.</returns>
public static double[] ExtractFields(EncogAnalyst analyst,
CSVHeaders headers, ReadCSV csv,
int outputLength, bool skipOutput)
{
var output = new double[outputLength];
int outputIndex = 0;
foreach (AnalystField stat in analyst.Script.Normalize.NormalizedFields)
{
if (stat.Action == NormalizationAction.Ignore)
{
continue;
}
if (stat.Output && skipOutput)
{
continue;
}
int index = headers.Find(stat.Name);
String str = csv.Get(index);
// is this an unknown value?
if (str.Equals("?") || str.Length == 0)
{
IHandleMissingValues handler = analyst.Script.Normalize.MissingValues;
double[] d = handler.HandleMissing(analyst, stat);
// should we skip the entire row
if (d == null)
{
return null;
}
// copy the returned values in place of the missing values
for (int i = 0; i < d.Length; i++)
{
output[outputIndex++] = d[i];
}
}
else
{
// known value
if (stat.Action == NormalizationAction.Normalize)
{
double d = csv.Format.Parse(str.Trim());
d = stat.Normalize(d);
output[outputIndex++] = d;
}
else
{
double[] d = stat.Encode(str.Trim());
foreach (double element in d)
{
output[outputIndex++] = element;
}
}
}
}
return output;
}
/// <summary>
/// Analyze the file.
/// </summary>
///
/// <param name="inputFilename">The input file.</param>
/// <param name="expectInputHeaders">True, if input headers are present.</param>
/// <param name="inputFormat">The format.</param>
/// <param name="theAnalyst">The analyst to use.</param>
public void Analyze(FileInfo inputFilename,
bool expectInputHeaders, CSVFormat inputFormat,
EncogAnalyst theAnalyst)
{
InputFilename = inputFilename;
InputFormat = inputFormat;
ExpectInputHeaders = expectInputHeaders;
_analyst = theAnalyst;
Analyzed = true;
_analystHeaders = new CSVHeaders(inputFilename, expectInputHeaders,
inputFormat);
foreach (AnalystField field in _analyst.Script.Normalize.NormalizedFields)
{
field.Init();
}
_series = new TimeSeriesUtil(_analyst,
_analystHeaders.Headers);
}
public static double[] ExtractFields(EncogAnalyst analyst, CSVHeaders headers, ReadCSV csv, int outputLength, bool skipOutput)
{
double[] numArray = new double[outputLength];
int num = 0;
using (IEnumerator<AnalystField> enumerator = analyst.Script.Normalize.NormalizedFields.GetEnumerator())
{
AnalystField field;
int num2;
string str;
IHandleMissingValues values;
double[] numArray2;
int num3;
double num4;
double num5;
double[] numArray4;
double[] numArray5;
int num6;
goto Label_0070;
Label_0022:
if (!skipOutput)
{
goto Label_02B7;
}
Label_0029:
if ((((uint) skipOutput) + ((uint) num6)) > uint.MaxValue)
{
goto Label_00B1;
}
goto Label_0070;
Label_0043:
if ((((uint) num4) - ((uint) outputLength)) > uint.MaxValue)
{
goto Label_0022;
}
if (((uint) num3) > uint.MaxValue)
{
goto Label_022A;
}
Label_0070:
if (enumerator.MoveNext())
{
goto Label_02CD;
}
goto Label_01C2;
Label_007E:
if (((uint) num2) > uint.MaxValue)
{
goto Label_021B;
}
goto Label_0043;
Label_009A:
if (!field.Output)
{
goto Label_02B7;
}
goto Label_0022;
Label_00B1:
num5 = numArray5[num6];
Label_00B9:
numArray[num++] = num5;
num6++;
Label_00C8:
if (num6 < numArray5.Length)
{
goto Label_00B1;
}
if ((((uint) num3) + ((uint) num5)) >= 0)
{
goto Label_0116;
}
Label_00E8:
numArray5 = field.Encode(str.Trim());
num6 = 0;
if ((((uint) num6) + ((uint) num5)) >= 0)
{
goto Label_00C8;
}
Label_0116:
if ((((uint) num4) - ((uint) num5)) <= uint.MaxValue)
{
goto Label_007E;
}
goto Label_01C2;
Label_0138:
if (field.Action == NormalizationAction.Normalize)
{
num4 = csv.Format.Parse(str.Trim());
num4 = field.Normalize(num4);
numArray[num++] = num4;
if ((((uint) outputLength) - ((uint) num6)) > uint.MaxValue)
{
goto Label_022A;
}
goto Label_0070;
}
if ((((uint) num2) - ((uint) outputLength)) <= uint.MaxValue)
{
goto Label_00E8;
}
goto Label_01C2;
Label_01A4:
if (num3 < numArray2.Length)
{
goto Label_0207;
}
goto Label_0070;
Label_01B1:
if (str.Length == 0)
{
goto Label_025F;
}
goto Label_0138;
Label_01C2:
if ((((uint) num6) - ((uint) num3)) >= 0)
{
return numArray;
}
if ((((uint) num2) - ((uint) num5)) <= uint.MaxValue)
{
goto Label_02CD;
}
goto Label_0295;
Label_0202:
num3 = 0;
goto Label_01A4;
Label_0207:
numArray[num++] = numArray2[num3];
num3++;
goto Label_02E6;
Label_021B:
numArray2 = values.HandleMissing(analyst, field);
if (numArray2 != null)
{
goto Label_0202;
}
Label_022A:
numArray4 = null;
if ((((uint) outputLength) | 8) != 0)
{
return numArray4;
}
goto Label_02E6;
Label_024E:
if (!str.Equals("?"))
{
goto Label_01B1;
}
Label_025F:
values = analyst.Script.Normalize.MissingValues;
goto Label_021B;
Label_0273:
if (((uint) num5) > uint.MaxValue)
{
goto Label_00B9;
}
str = csv.Get(num2);
goto Label_024E;
Label_0295:
if ((((uint) num5) | uint.MaxValue) == 0)
{
goto Label_0029;
}
goto Label_0022;
Label_02B7:
num2 = headers.Find(field.Name);
goto Label_0273;
Label_02CD:
field = enumerator.Current;
if (field.Action == NormalizationAction.Ignore)
{
goto Label_0070;
}
goto Label_009A;
Label_02E6:
if ((((uint) num4) - ((uint) skipOutput)) >= 0)
{
goto Label_01A4;
}
}
return numArray;
}
public void Analyze(FileInfo inputFilename, bool expectInputHeaders, CSVFormat inputFormat, EncogAnalyst theAnalyst)
{
base.InputFilename = inputFilename;
if (1 != 0)
{
base.InputFormat = inputFormat;
base.ExpectInputHeaders = expectInputHeaders;
this._x554f16462d8d4675 = theAnalyst;
base.Analyzed = true;
this._xc5416b6511261016 = new CSVHeaders(inputFilename, expectInputHeaders, inputFormat);
foreach (AnalystField field in this._x554f16462d8d4675.Script.Normalize.NormalizedFields)
{
field.Init();
}
}
this._x7acb8518c8ed6133 = new TimeSeriesUtil(this._x554f16462d8d4675, true, this._xc5416b6511261016.Headers);
if (1 != 0)
{
}
}
/// <summary>
/// Generate the fields using header values.
/// </summary>
/// <param name="csv">The CSV file to use.</param>
private void GenerateFieldsFromHeaders(ReadCSV csv)
{
var h = new CSVHeaders(csv.ColumnNames);
_fields = new AnalyzedField[csv.ColumnCount];
for (int i = 0; i < _fields.Length; i++)
{
if (i >= csv.ColumnCount)
{
throw new AnalystError(
"CSV header count does not match column count");
}
_fields[i] = new AnalyzedField(_script, h.GetHeader(i));
}
}
public void Analyze(EncogAnalyst theAnalyst, FileInfo inputFile, bool headers, CSVFormat format)
{
int num;
int num2;
ReadCSV dcsv;
base.InputFilename = inputFile;
if (0 == 0)
{
if ((((uint) headers) + ((uint) num2)) < 0)
{
goto Label_00CC;
}
base.ExpectInputHeaders = headers;
base.InputFormat = format;
if ((((uint) num) + ((uint) headers)) >= 0)
{
base.Analyzed = true;
this._x554f16462d8d4675 = theAnalyst;
if (base.OutputFormat == null)
{
base.OutputFormat = base.InputFormat;
if (((uint) num2) < 0)
{
goto Label_007E;
}
}
}
}
goto Label_0184;
Label_0044:
base.RecordCount = num;
base.Count = dcsv.ColumnCount;
base.ReadHeaders(dcsv);
dcsv.Close();
base.ReportDone(true);
if ((((uint) num2) | 0xfffffffe) != 0)
{
return;
}
goto Label_0184;
Label_0074:
if (!base.ShouldStop())
{
base.UpdateStatus(true);
LoadedRow theRow = new LoadedRow(dcsv, 1);
double[] input = AnalystNormalizeCSV.ExtractFields(this._x554f16462d8d4675, this._xc5416b6511261016, dcsv, num2, true);
if ((((uint) num2) + ((uint) num)) >= 0)
{
ClusterRow inputData = new ClusterRow(input, theRow);
this._x4a3f0a05c02f235f.Add(inputData);
if ((((uint) num2) + ((uint) num2)) >= 0)
{
num++;
if ((((uint) num2) & 0) == 0)
{
goto Label_007E;
}
goto Label_0074;
}
goto Label_00C5;
}
goto Label_011C;
}
goto Label_0044;
Label_007E:
if (dcsv.Next())
{
goto Label_0074;
}
goto Label_0044;
Label_00C5:
if (2 == 0)
{
goto Label_0074;
}
Label_00CC:
this._xc5416b6511261016 = new CSVHeaders(base.InputHeadings);
goto Label_007E;
Label_011C:
num2 = this._x554f16462d8d4675.DetermineTotalColumns();
dcsv = new ReadCSV(base.InputFilename.ToString(), base.ExpectInputHeaders, base.InputFormat);
base.ReadHeaders(dcsv);
goto Label_00C5;
Label_0184:
this._x4a3f0a05c02f235f = new BasicMLDataSet();
base.ResetStatus();
if ((((uint) headers) - ((uint) num)) > uint.MaxValue)
{
goto Label_0044;
}
num = 0;
goto Label_011C;
}
private int[] x163e1f9de31a8b41(CSVHeaders x36c9f86d45fbd962)
{
int num;
string str;
int slice;
AnalystField field;
int[] numArray;
int num3;
IList<int> list = new List<int>();
if (((uint) slice) >= 0)
{
goto Label_0155;
}
if ((((uint) num) - ((uint) num)) <= uint.MaxValue)
{
goto Label_003D;
}
Label_0033:
while (num3 < numArray.Length)
{
numArray[num3] = list[num3];
num3++;
if (((uint) slice) < 0)
{
return numArray;
}
}
return numArray;
Label_003D:
numArray = new int[list.Count];
num3 = 0;
if ((((uint) num3) - ((uint) slice)) > uint.MaxValue)
{
goto Label_015C;
}
goto Label_0033;
Label_0094:
if ((((uint) num) & 0) != 0)
{
goto Label_00ED;
}
if ((((uint) num) + ((uint) slice)) > uint.MaxValue)
{
return numArray;
}
Label_00C3:
num++;
if ((((uint) num3) + ((uint) num)) < 0)
{
goto Label_00C3;
}
Label_00DF:
if (num < x36c9f86d45fbd962.Size())
{
goto Label_015C;
}
goto Label_003D;
Label_00ED:
if (field == null)
{
goto Label_00C3;
}
if (!field.Input)
{
if (((uint) num) <= uint.MaxValue)
{
goto Label_00C3;
}
goto Label_0094;
}
Label_011F:
list.Add(num);
if (0 == 0)
{
goto Label_0094;
}
Label_0155:
num = 0;
goto Label_00DF;
Label_015C:
str = x36c9f86d45fbd962.GetBaseHeader(num);
if ((((uint) num3) + ((uint) slice)) >= 0)
{
slice = x36c9f86d45fbd962.GetSlice(num);
field = base.Analyst.Script.FindNormalizedField(str, slice);
goto Label_00ED;
}
if (-1 == 0)
{
goto Label_00C3;
}
if ((((uint) num) + ((uint) num)) >= 0)
{
goto Label_011F;
}
goto Label_0094;
}
private int[] x176a88b9713cb7be(CSVHeaders x36c9f86d45fbd962)
{
IList<int> list;
int num;
int slice;
AnalystField field;
string propertyString = base.Prop.GetPropertyString("ML:CONFIG_type");
if ((((uint) slice) & 0) == 0)
{
if (2 == 0)
{
goto Label_007D;
}
if (propertyString.Equals("som"))
{
return new int[0];
}
list = new List<int>();
num = 0;
}
else
{
goto Label_0079;
}
Label_005D:
if (num < x36c9f86d45fbd962.Size())
{
string baseHeader = x36c9f86d45fbd962.GetBaseHeader(num);
slice = x36c9f86d45fbd962.GetSlice(num);
field = base.Analyst.Script.FindNormalizedField(baseHeader, slice);
}
else
{
int[] numArray = new int[list.Count];
int index = 0;
do
{
if (index >= numArray.Length)
{
return numArray;
}
numArray[index] = list[index];
index++;
}
while ((((uint) index) - ((uint) num)) <= uint.MaxValue);
goto Label_005D;
}
Label_0079:
if ((field != null) && field.Output)
{
list.Add(num);
}
Label_007D:
num++;
goto Label_005D;
}
