/// <summary>
/// Get an array of ints's from a string of comma separated text.
/// </summary>
/// <param name="format">The way to format this list.</param>
/// <param name="str">The string that contains a list of numbers.</param>
/// <returns>An array of ints parsed from the string.</returns>
public static int[] FromListInt(CSVFormat format, String str)
{
if (str.Trim().Length == 0)
{
return(new int[0]);
}
// first count the numbers
String[] tok = str.Split(format.Separator);
int count = tok.Length;
// now allocate an object to hold that many numbers
var result = new int[count];
// and finally parse the numbers
for (int index = 0; index < tok.Length; index++)
{
try
{
String num = tok[index];
int value = int.Parse(num);
result[index] = value;
}
catch (Exception e)
{
throw new PersistError(e);
}
}
return(result);
}
/// <summary>
/// Load a CSV file into a memory dataset.
/// </summary>
///
/// <param name="format">The CSV format to use.</param>
/// <param name="filename">The filename to load.</param>
/// <param name="headers">True if there is a header line.</param>
/// <param name="inputSize">The input size. Input always comes first in a file.</param>
/// <param name="idealSize">The ideal size, 0 for unsupervised.</param>
/// <returns>A NeuralDataSet that holds the contents of the CSV file.</returns>
public static IMLDataSet LoadCSVTOMemory(CSVFormat format, String filename,
bool headers, int inputSize, int idealSize)
{
var result = new BasicMLDataSet();
var csv = new ReadCSV(filename, headers, format);
while (csv.Next())
{
BasicMLData ideal = null;
int index = 0;
var input = new BasicMLData(inputSize);
for (int i = 0; i < inputSize; i++)
{
double d = csv.GetDouble(index++);
input[i] = d;
}
if (idealSize > 0)
{
ideal = new BasicMLData(idealSize);
for (int i = 0; i < idealSize; i++)
{
double d = csv.GetDouble(index++);
ideal[i] = d;
}
}
IMLDataPair pair = new BasicMLDataPair(input, ideal);
result.Add(pair);
}
return result;
}
public void Analyze(EncogAnalyst theAnalyst, FileInfo inputFile, bool headers, CSVFormat format)
{
base.InputFilename = inputFile;
if ((((uint) headers) & 0) != 0)
{
goto Label_0063;
}
Label_005C:
base.ExpectInputHeaders = headers;
Label_0063:
base.InputFormat = format;
base.Analyzed = true;
this._x554f16462d8d4675 = theAnalyst;
base.PerformBasicCounts();
if (((uint) headers) >= 0)
{
this._x146688677da5adf5 = base.InputHeadings.Length;
this._x1402a42b31a31090 = this._x554f16462d8d4675.DetermineOutputFieldCount();
this._xc5416b6511261016 = new CSVHeaders(base.InputHeadings);
this._x7acb8518c8ed6133 = new TimeSeriesUtil(this._x554f16462d8d4675, false, this._xc5416b6511261016.Headers);
if (0 != 0)
{
goto Label_005C;
}
}
}
/// <summary>
/// Construct a CSV source from a filename. Allows a delimiter character to
/// be specified.
/// </summary>
/// <param name="file">The filename.</param>
/// <param name="headers">The headers.</param>
/// <param name="format">The format.</param>
public CSVDataSource(string file, bool headers,
CSVFormat format)
{
_file = file;
_headers = headers;
_format = format;
}
/// <summary>
/// Read the headers.
/// </summary>
/// <param name="format">The format of this CSV file.</param>
/// <param name="headers">Are headers present.</param>
private void Begin(bool headers, CSVFormat format)
{
try
{
this.format = format;
// read the column heads
if (headers)
{
String line = this.reader.ReadLine();
IList <String> tok = Parse(line);
int i = 0;
foreach (String header in tok)
{
this.columns.Add(header.ToLower(), i++);
}
}
this.data = null;
}
catch (IOException e)
{
#if logging
if (this.logger.IsErrorEnabled)
{
this.logger.Error("Exception", e);
}
#endif
throw new EncogError(e);
}
}
/// <summary>
/// Analyze the data. This counts the records and prepares the data to be
/// processed.
/// </summary>
/// <param name="theAnalyst">The analyst to use.</param>
/// <param name="inputFile">The input file.</param>
/// <param name="headers">True if headers are present.</param>
/// <param name="format">The format the file is in.</param>
public void Analyze(EncogAnalyst theAnalyst,
FileInfo inputFile, bool headers, CSVFormat format)
{
InputFilename = inputFile;
ExpectInputHeaders = headers;
Format = format;
_analyst = theAnalyst;
Analyzed = true;
PerformBasicCounts();
_inputCount = _analyst.DetermineInputCount();
_outputCount = _analyst.DetermineOutputCount();
_idealCount = InputHeadings.Length - _inputCount;
if ((InputHeadings.Length != _inputCount)
&& (InputHeadings.Length != (_inputCount + _outputCount)))
{
throw new AnalystError("Invalid number of columns("
+ InputHeadings.Length + "), must match input("
+ _inputCount + ") count or input+output("
+ (_inputCount + _outputCount) + ") count.");
}
}
public CSVDataCODEC(string file, CSVFormat format, bool headers, int inputCount, int idealCount, bool significance)
{
if ((((uint) inputCount) - ((uint) significance)) < 0)
{
}
Label_0063:
if (this._x43f451310e815b76 != 0)
{
throw new BufferedDataError("To export CSV, you must use the CSVDataCODEC constructor that does not specify input or ideal sizes.");
}
Label_006B:
this._xb44380e048627945 = file;
if ((((uint) inputCount) - ((uint) inputCount)) <= uint.MaxValue)
{
this._x5786461d089b10a0 = format;
this._x43f451310e815b76 = inputCount;
this._xb52d4a98fad404da = idealCount;
this._x94e6ca5ac178dbd0 = headers;
this._x2602a84fb5c05ca2 = significance;
if ((((uint) headers) - ((uint) inputCount)) > uint.MaxValue)
{
goto Label_0063;
if ((((uint) significance) + ((uint) headers)) >= 0)
{
goto Label_006B;
}
goto Label_0063;
}
}
}
public ReadCSV(string filename, bool headers, CSVFormat format)
{
this._x44b48f93f08f9199 = new List<string>();
this._x26c511b92db96554 = new Dictionary<string, int>();
this._xe134235b3526fa75 = new StreamReader(filename);
this.x5e9e6297e1dda8c2(headers, format);
}
/// <summary>
/// Get an array of ints's from a string of comma separated text.
/// </summary>
/// <param name="format">The way to format this list.</param>
/// <param name="str">The string that contains a list of numbers.</param>
/// <returns>An array of ints parsed from the string.</returns>
public static int[] FromListInt(CSVFormat format, String str)
{
if (str.Trim().Length == 0)
{
return new int[0];
}
// first count the numbers
String[] tok = str.Split(format.Separator);
int count = tok.Length;
// now allocate an object to hold that many numbers
var result = new int[count];
// and finally parse the numbers
for (int index = 0; index < tok.Length; index++)
{
try
{
String num = tok[index];
int value = int.Parse(num);
result[index] = value;
}
catch (Exception e)
{
throw new PersistError(e);
}
}
return result;
}
/// <summary>
/// Get an array of double's from a string of comma separated text.
/// </summary>
/// <param name="format">The way to format this list.</param>
/// <param name="str">The string that contains a list of numbers.</param>
/// <returns>An array of doubles parsed from the string.</returns>
public static double[] FromList(CSVFormat format, String str)
{
// first count the numbers
String[] tok = str.Split(format.Separator);
int count = tok.Length;
// now allocate an object to hold that many numbers
double[] result = new double[count];
// and finally parse the numbers
for (int index = 0; index < tok.Length; index++)
{
try
{
String num = tok[index];
double value = format.Parse(num);
result[index] = value;
}
catch (Exception e)
{
throw new PersistError(e);
}
}
return result;
}
/// <summary>
/// Construct an analyzed field.
/// </summary>
/// <param name="theScript">The script being analyzed.</param>
/// <param name="name">The name of the field.</param>
public AnalyzedField(AnalystScript theScript, String name) : base(name)
{
_classMap = new Dictionary<String, AnalystClassItem>();
_instances = 0;
_script = theScript;
_fmt = _script.DetermineFormat();
}
/// <summary>
/// Get an array of double's from a string of comma separated text.
/// </summary>
/// <param name="format">The way to format this list.</param>
/// <param name="str">The string that contains a list of numbers.</param>
/// <returns>An array of doubles parsed from the string.</returns>
public static double[] FromList(CSVFormat format, String str)
{
// first count the numbers
String[] tok = str.Split(format.Separator);
int count = tok.Length;
// now allocate an object to hold that many numbers
double[] result = new double[count];
// and finally parse the numbers
for (int index = 0; index < tok.Length; index++)
{
try
{
String num = tok[index];
double value = format.Parse(num);
result[index] = value;
}
catch (Exception e)
{
throw new PersistError(e);
}
}
return(result);
}
public void Analyze(FileInfo inputFile, bool headers, CSVFormat format)
{
base.InputFilename = inputFile;
base.ExpectInputHeaders = headers;
base.InputFormat = format;
base.Analyzed = true;
base.PerformBasicCounts();
}
/// <summary>
/// Construct a CSV reader from a filename.
/// </summary>
/// <param name="filename">The filename.</param>
/// <param name="headers">The headers.</param>
/// <param name="delim">The delimiter.</param>
public ReadCSV(String filename, bool headers,
char delim)
{
var format = new CSVFormat(CSVFormat.DecimalCharacter, delim);
_reader = new StreamReader(filename);
_delim = delim;
Begin(headers, format);
}
/// <summary>
/// Construct a CSV reader from an input stream.
/// </summary>
/// <param name="istream">The InputStream to read from.</param>
/// <param name="headers">Are headers present?</param>
/// <param name="delim">What is the delimiter.</param>
public ReadCSV(Stream istream, bool headers,
char delim)
{
var format = new CSVFormat(CSVFormat.DecimalCharacter, delim);
_reader = new StreamReader(istream);
_delim = delim;
Begin(headers, format);
}
/// <summary>
/// Construct a CSV source from a filename. The format parameter specifies
/// the separator character to use, as well as the number format.
/// </summary>
/// <param name="file">The filename.</param>
/// <param name="headers">The headers.</param>
/// <param name="delim">The delimiter.</param>
public CSVDataSource(string file, bool headers,
char delim)
{
_format = new CSVFormat(CSVFormat.DecimalCharacter,
delim);
_headers = headers;
_file = file;
}
/// <summary>
/// Construct a loaded row from an array.
/// </summary>
/// <param name="format">The format to store the numbers in.</param>
/// <param name="data">The data to use.</param>
/// <param name="extra">The extra positions to allocate.</param>
public LoadedRow(CSVFormat format, double[] data, int extra)
{
int count = data.Length;
_data = new String[count + extra];
for (int i = 0; i < count; i++)
{
_data[i] = format.Format(data[i], 5);
}
}
/// <summary>
/// Construct a loaded row from an IMLData.
/// </summary>
/// <param name="format">The format to store the numbers in.</param>
/// <param name="data">The data to use.</param>
/// <param name="extra">The extra positions to allocate.</param>
public LoadedRow(CSVFormat format, IMLData data, int extra)
{
int count = data.Count;
_data = new String[count + extra];
for (int i = 0; i < count; i++)
{
_data[i] = format.Format(data[i], 5);
}
}
public ReadCSV(string filename, bool headers, char delim)
{
this._x44b48f93f08f9199 = new List<string>();
this._x26c511b92db96554 = new Dictionary<string, int>();
CSVFormat format = new CSVFormat(CSVFormat.DecimalCharacter, delim);
this._xe134235b3526fa75 = new StreamReader(filename);
this._x150fef4785a04465 = delim;
this.x5e9e6297e1dda8c2(headers, format);
}
/// <summary>
/// Construct a CSV reader from an input stream.
/// </summary>
/// <param name="istream">The InputStream to read from.</param>
/// <param name="headers">Are headers present?</param>
/// <param name="delim">What is the delimiter.</param>
public ReadCSV(Stream istream, bool headers,
char delim)
{
CSVFormat format = new CSVFormat(CSVFormat.DecimalCharacter, delim);
this.reader = new StreamReader(istream);
this.delim = delim;
Begin(headers, format);
}
public void Analyze(IMLRegression method, FileInfo inputFile, bool headers, CSVFormat format)
{
object[] objArray;
base.InputFilename = inputFile;
base.ExpectInputHeaders = headers;
base.InputFormat = format;
if (((uint) headers) >= 0)
{
base.Analyzed = true;
}
base.PerformBasicCounts();
this._x43f451310e815b76 = method.InputCount;
this._x98cf41c6b0eaf6ab = method.OutputCount;
this._xb52d4a98fad404da = Math.Max(base.InputHeadings.Length - this._x43f451310e815b76, 0);
if (0x7fffffff == 0)
{
goto Label_00E0;
}
if ((((uint) headers) & 0) != 0)
{
goto Label_0084;
}
if ((((uint) headers) - ((uint) headers)) >= 0)
{
if (base.InputHeadings.Length == this._x43f451310e815b76)
{
return;
}
if (3 == 0)
{
return;
}
}
Label_000C:
if (base.InputHeadings.Length != (this._x43f451310e815b76 + this._x98cf41c6b0eaf6ab))
{
objArray = new object[7];
goto Label_00E0;
}
return;
Label_0084:
objArray[3] = this._x43f451310e815b76;
if ((((uint) headers) + ((uint) headers)) < 0)
{
goto Label_000C;
}
objArray[4] = ") count or input+output(";
objArray[5] = this._x43f451310e815b76 + this._x98cf41c6b0eaf6ab;
objArray[6] = ") count.";
throw new AnalystError(string.Concat(objArray));
Label_00E0:
objArray[0] = "Invalid number of columns(";
objArray[1] = base.InputHeadings.Length;
objArray[2] = "), must match input(";
goto Label_0084;
}
/// <summary>
/// parses one column of a csv and returns an array of doubles.
/// you can only return one double array with this method.
/// </summary>
/// <param name="file">The file.</param>
/// <param name="formatused">The formatused.</param>
/// <param name="Name">The name of the column to parse..</param>
/// <returns></returns>
public static List<double> QuickParseCSV(string file, CSVFormat formatused, string Name)
{
List<double> returnedArrays = new List<double>();
ReadCSV csv = new ReadCSV(file, true, formatused);
while (csv.Next())
{
returnedArrays.Add(csv.GetDouble(Name));
}
return returnedArrays;
}
/// <summary>
/// Analyze the data. This counts the records and prepares the data to be
/// processed.
/// </summary>
/// <param name="inputFile">The input file to process.</param>
/// <param name="headers">True, if headers are present.</param>
/// <param name="format">The format of the CSV file.</param>
public void Analyze(FileInfo inputFile, bool headers,
CSVFormat format)
{
InputFilename = inputFile;
ExpectInputHeaders = headers;
Format = format;
Analyzed = true;
PerformBasicCounts();
}
/// <summary>
/// Process, and sort the files.
/// </summary>
///
/// <param name="inputFile">The input file.</param>
/// <param name="outputFile">The output file.</param>
/// <param name="headers">True, if headers are to be used.</param>
/// <param name="format">The format of the file.</param>
public void Process(FileInfo inputFile, FileInfo outputFile,
bool headers, CSVFormat format)
{
InputFilename = inputFile;
ExpectInputHeaders = headers;
InputFormat = format;
ReadInputFile();
SortData();
WriteOutputFile(outputFile);
}
public void Process(FileInfo inputFile, FileInfo outputFile, bool headers, CSVFormat format)
{
base.InputFilename = inputFile;
base.ExpectInputHeaders = headers;
base.InputFormat = format;
this.xcc7d420ca2a80044();
do
{
this.x116e636386480d14();
this.x64b4cdf32944d344(outputFile);
}
while ((((uint) headers) | 1) == 0);
}
/// <summary>
/// Convert an IMLData to a comma separated list.
/// </summary>
/// <param name="format">The way to format this list.</param>
/// <param name="result">This string will have the values appended to it.</param>
/// <param name="data">The array of doubles to use.</param>
public static void ToList(CSVFormat format, StringBuilder result,
IMLData data)
{
result.Length = 0;
for (int i = 0; i < data.Count; i++)
{
if (i != 0)
{
result.Append(format.Separator);
}
result.Append(format.Format(data[i], EncogFramework.DefaultPrecision));
}
}
/// <summary>
/// Convert an array of doubles to a comma separated list.
/// </summary>
/// <param name="format">The way to format this list.</param>
/// <param name="result">This string will have the values appended to it.</param>
/// <param name="data">The array of doubles to use.</param>
public static void ToList(CSVFormat format, StringBuilder result,
double[] data)
{
result.Length = 0;
for (int i = 0; i < data.Length; i++)
{
if (i != 0)
{
result.Append(format.Separator);
}
result.Append(format.Format(data[i], 20));
}
}
public void Analyze(FileInfo inputFile, bool headers, CSVFormat format)
{
base.InputFilename = inputFile;
do
{
base.ExpectInputHeaders = headers;
base.InputFormat = format;
}
while ((((uint) headers) + ((uint) headers)) < 0);
base.Analyzed = true;
base.PerformBasicCounts();
this.xae1ebc39c039dcb2();
}
/// <summary>
/// Convert a CSV file to a binary training file.
/// </summary>
/// <param name="csvFile">The CSV file.</param>
/// <param name="format">The format.</param>
/// <param name="binFile">The binary file.</param>
/// <param name="inputCount">The number of input values.</param>
/// <param name="outputCount">The number of output values.</param>
/// <param name="headers">True, if there are headers on the3 CSV.</param>
/// <param name="expectSignificance">Should a significance column be expected.</param>
public static void ConvertCSV2Binary(String csvFile, CSVFormat format,
String binFile, int inputCount, int outputCount,
bool headers, bool expectSignificance)
{
new FileInfo(binFile).Delete();
var csv = new CSVMLDataSet(csvFile,
inputCount, outputCount, false, format, expectSignificance);
var buffer = new BufferedMLDataSet(binFile);
buffer.BeginLoad(inputCount, outputCount);
foreach (IMLDataPair pair in csv)
{
buffer.Add(pair);
}
buffer.EndLoad();
}
public CSVMLDataSet(string filename, int inputSize, int idealSize, bool headers, CSVFormat format, bool expectSignificance)
{
this._xb41a802ca5fde63b = filename;
this._x7e648b416c264559 = inputSize;
if (-2 != 0)
{
this._x08b9e0820ab2b457 = idealSize;
}
this._x5786461d089b10a0 = format;
this._x94e6ca5ac178dbd0 = headers;
IDataSetCODEC codec = new CSVDataCODEC(filename, format, headers, inputSize, idealSize, expectSignificance);
MemoryDataLoader loader2 = new MemoryDataLoader(codec) {
Result = this
};
loader2.External2Memory();
}
public static void ToList(CSVFormat format, int precision, StringBuilder result, double[] data)
{
int num;
result.Length = 0;
if ((((uint) precision) + ((uint) num)) <= uint.MaxValue)
{
goto Label_007B;
}
if (0 == 0)
{
goto Label_0064;
}
Label_0022:
if ((((uint) num) & 0) != 0)
{
goto Label_0064;
}
Label_0036:
result.Append(format.Format(data[num], precision));
num++;
Label_004B:
if (num < data.Length)
{
goto Label_0064;
}
if (((uint) num) >= 0)
{
return;
}
goto Label_007B;
Label_0055:
result.Append(format.Separator);
goto Label_0022;
Label_0064:
if (num != 0)
{
goto Label_0055;
}
goto Label_0036;
Label_007B:
num = 0;
if (0 != 0)
{
goto Label_0055;
}
goto Label_004B;
}
public void Analyze(EncogAnalyst theAnalyst, FileInfo inputFile, bool headers, CSVFormat format)
{
object[] objArray;
base.InputFilename = inputFile;
base.ExpectInputHeaders = headers;
base.InputFormat = format;
this._x554f16462d8d4675 = theAnalyst;
if ((((uint) headers) - ((uint) headers)) >= 0)
{
base.Analyzed = true;
if (0 == 0)
{
base.PerformBasicCounts();
this._x43f451310e815b76 = this._x554f16462d8d4675.DetermineInputCount();
this._x98cf41c6b0eaf6ab = this._x554f16462d8d4675.DetermineOutputCount();
this._xb52d4a98fad404da = base.InputHeadings.Length - this._x43f451310e815b76;
while ((base.InputHeadings.Length != this._x43f451310e815b76) && (base.InputHeadings.Length != (this._x43f451310e815b76 + this._x98cf41c6b0eaf6ab)))
{
objArray = new object[7];
if (0 != 0)
{
goto Label_0064;
}
objArray[0] = "Invalid number of columns(";
if (0 == 0)
{
objArray[1] = base.InputHeadings.Length;
objArray[2] = "), must match input(";
objArray[3] = this._x43f451310e815b76;
objArray[4] = ") count or input+output(";
objArray[5] = this._x43f451310e815b76 + this._x98cf41c6b0eaf6ab;
goto Label_005C;
}
if (0x7fffffff == 0)
{
return;
}
}
return;
}
return;
}
Label_005C:
objArray[6] = ") count.";
Label_0064:
throw new AnalystError(string.Concat(objArray));
}
/// <summary>
/// Analyze the data. This counts the records and prepares the data to be
/// processed.
/// </summary>
///
/// <param name="theAnalyst">The analyst to use.</param>
/// <param name="inputFile">The input file to analyze.</param>
/// <param name="headers">True, if the input file has headers.</param>
/// <param name="format">The format of the input file.</param>
public void Analyze(EncogAnalyst theAnalyst,
FileInfo inputFile, bool headers, CSVFormat format)
{
InputFilename = inputFile;
ExpectInputHeaders = headers;
InputFormat = format;
Analyzed = true;
_analyst = theAnalyst;
if (OutputFormat == null)
{
OutputFormat = InputFormat;
}
_data = new BasicMLDataSet();
ResetStatus();
int recordCount = 0;
int outputLength = _analyst.DetermineTotalColumns();
var csv = new ReadCSV(InputFilename.ToString(),
ExpectInputHeaders, InputFormat);
ReadHeaders(csv);
_analystHeaders = new CSVHeaders(InputHeadings);
while (csv.Next() && !ShouldStop())
{
UpdateStatus(true);
var row = new LoadedRow(csv, 1);
double[] inputArray = AnalystNormalizeCSV.ExtractFields(
_analyst, _analystHeaders, csv, outputLength, true);
var input = new ClusterRow(inputArray, row);
_data.Add(input);
recordCount++;
}
RecordCount = recordCount;
Count = csv.ColumnCount;
ReadHeaders(csv);
csv.Close();
ReportDone(true);
}
static void Main(string[] args)
{
var format = new CSVFormat('.', ',');
IVersatileDataSource source = new CSVDataSource("input.csv", true, format);
var data = new VersatileMLDataSet(source);
data.NormHelper.Format = format;
for (int i = 0; i < 228; i++)
{
data.DefineSourceColumn("a" + (i + 1), i, ColumnType.Continuous);
}
ColumnDefinition columnOutput = data.DefineSourceColumn("close", 228, ColumnType.Continuous);
data.Analyze();
data.DefineSingleOutputOthersInput(columnOutput);
var model = new EncogModel(data);
model.SelectMethod(data, MLMethodFactory.TypeFeedforward);
model.Report = new ConsoleStatusReportable();
data.LeadWindowSize = 1;
data.LagWindowSize = 2;
data.Normalize();
model.HoldBackValidation(0.3, true, 1001);
model.SelectTrainingType(data);
// Use a 5-fold cross-validated train. Return the best method found.
var bestMethod = (IMLRegression)model.Crossvalidate(5, true);
// Display the training and validation errors.
Console.WriteLine(@"Training error: " + model.CalculateError(bestMethod, model.TrainingDataset));
Console.WriteLine(@"Validation error: " + model.CalculateError(bestMethod, model.ValidationDataset));
// Display our normalization parameters.
NormalizationHelper helper = data.NormHelper;
Console.WriteLine(helper.ToString());
// Display the final model.
Console.WriteLine(@"Final model: " + bestMethod);
}
/// <summary>
/// Create a CODEC to load data from CSV to binary.
/// </summary>
/// <param name="file">The CSV file to load.</param>
/// <param name="format">The format that the CSV file is in.</param>
/// <param name="headers">True, if there are headers.</param>
/// <param name="inputCount">The number of input columns.</param>
/// <param name="idealCount">The number of ideal columns.</param>
public CSVDataCODEC(
String file,
CSVFormat format,
bool headers,
int inputCount, int idealCount)
{
if (this.inputCount != 0)
{
throw new BufferedDataError(
"To export CSV, you must use the CSVDataCODEC constructor that does not specify input or ideal sizes.");
}
this.file = file;
this.format = format;
this.inputCount = inputCount;
this.idealCount = idealCount;
this.headers = headers;
}
/// <summary>
/// Analyze the data. This counts the records and prepares the data to be
/// processed.
/// </summary>
///
/// <param name="theAnalyst">The analyst to use.</param>
/// <param name="inputFile">The input file.</param>
/// <param name="headers">True if headers are present.</param>
/// <param name="format">The format.</param>
public void Analyze(EncogAnalyst theAnalyst,
FileInfo inputFile, bool headers, CSVFormat format)
{
InputFilename = inputFile;
ExpectInputHeaders = headers;
Format = format;
Analyzed = true;
_analyst = theAnalyst;
PerformBasicCounts();
_fileColumns = InputHeadings.Length;
_outputColumns = _analyst.DetermineOutputFieldCount();
_analystHeaders = new CSVHeaders(InputHeadings);
_series = new TimeSeriesUtil(_analyst, false,
_analystHeaders.Headers);
}
/// <summary>
/// Analyze the neural network.
/// </summary>
/// <param name="inputFile">The input file.</param>
/// <param name="headers">True, if there are headers.</param>
/// <param name="format">The format of the CSV file.</param>
public void Analyze(FileInfo inputFile, bool headers, CSVFormat format)
{
InputFilename = inputFile;
ExpectInputHeaders = headers;
Format = format;
Analyzed = true;
PerformBasicCounts();
expressionFields.Clear();
extension = new ProcessExtension(Format);
extension.register(programContext.Functions);
foreach (ProcessField field in analyst.Script.Process.Fields)
{
var prg = new EncogProgram(programContext, programVariables);
prg.SetExtraData(ProcessExtension.EXTENSION_DATA_NAME, extension);
prg.CompileExpression(field.Command);
expressionFields.Add(prg);
}
}
/// <summary>
/// Read the headers.
/// </summary>
/// <param name="format">The format of this CSV file.</param>
/// <param name="headers">Are headers present.</param>
private void Begin(bool headers, CSVFormat format)
{
try
{
DateFormat = "yyyy-MM-dd";
TimeFormat = "HHmmss";
this.parseLine = new ParseCSVLine(format);
// read the column heads
if (headers)
{
String line = _reader.ReadLine();
IList <String> tok = parseLine.Parse(line);
int i = 0;
foreach (String header in tok)
{
if (_columns.ContainsKey(header.ToLower()))
{
throw new EncogError("Two columns cannot have the same name");
}
_columns.Add(header.ToLower(), i++);
_columnNames.Add(header);
}
}
_data = null;
}
catch (IOException e)
{
#if logging
if (logger.IsErrorEnabled)
{
logger.Error("Exception", e);
}
#endif
throw new EncogError(e);
}
}
/// <summary>
/// Convert an array of doubles to a comma separated list.
/// </summary>
/// <param name="format">The way to format this list.</param>
/// <param name="result">This string will have the values appended to it.</param>
/// <param name="data">The array of doubles to use.</param>
public static void ToList(CSVFormat format, StringBuilder result,
double[] data)
{
ToList(format, 20, result, data);
}
/// <summary>
/// Construct a CSV reader from an input stream.
/// The format parameter specifies the separator
/// character to use, as well as the number
/// format.
/// </summary>
/// <param name="stream">The Stream to read from.</param>
/// <param name="headers">Are headers present?</param>
/// <param name="format">What is the CSV format.</param>
public ReadCSV(Stream stream, bool headers,
CSVFormat format)
{
_reader = new StreamReader(stream);
Begin(headers, format);
}
/// <summary>
/// Construct a CSV reader from a filename.
/// Allows a delimiter character to be specified.
/// Numbers will be parsed using the current
/// locale.
/// </summary>
/// <param name="filename">The filename.</param>
/// <param name="headers">The headers.</param>
/// <param name="format">The delimiter.</param>
public ReadCSV(String filename, bool headers,
CSVFormat format)
{
_reader = new StreamReader(filename);
Begin(headers, format);
}
static CSVFormat()
{
DecimalComma = new CSVFormat(',', ';');
DecimalPoint = new CSVFormat('.', ',');
}
public ParseCSVLine(CSVFormat theFormat)
{
Format = theFormat;
}