/// <summary>
/// Read the input file.
/// </summary>
private void ReadInputFile()
{
ResetStatus();
var csv = new ReadCSV(InputFilename.ToString(),
ExpectInputHeaders, Format);
while (csv.Next() && !ShouldStop())
{
UpdateStatus("Reading input file");
var row = new LoadedRow(csv);
_data.Add(row);
}
Count = csv.ColumnCount;
if (ExpectInputHeaders)
{
InputHeadings = new String[csv.ColumnCount];
for (int i = 0; i < csv.ColumnCount; i++)
{
InputHeadings[i] = csv.ColumnNames[i];
}
}
csv.Close();
}
/// <summary>
/// Process, and generate the output file.
/// </summary>
/// <param name="outputFile">The output file.</param>
public void Process(FileInfo outputFile)
{
ValidateAnalyzed();
var csv = new ReadCSV(InputFilename.ToString(), ExpectInputHeaders, Format);
LoadedRow row;
StreamWriter tw = PrepareOutputFile(outputFile);
extension.Init(csv,
forwardWindowSize,
backwardWindowSize);
ResetStatus();
while ((row = GetNextRow(csv)) != null)
{
extension.LoadRow(row);
if (extension.IsDataReady())
{
ProcessRow(tw);
}
UpdateStatus(false);
}
ReportDone(false);
tw.Close();
}
/// <summary>
/// Process the input file and segregate into the output files.
/// </summary>
///
public void Process()
{
Validate();
var csv = new ReadCSV(InputFilename.ToString(),
ExpectInputHeaders, InputFormat);
ResetStatus();
foreach (SegregateTargetPercent target in _targets)
{
StreamWriter tw = PrepareOutputFile(target.Filename);
while ((target.NumberRemaining > 0) && csv.Next() &&
!ShouldStop())
{
UpdateStatus(false);
var row = new LoadedRow(csv);
WriteRow(tw, row);
target.NumberRemaining = target.NumberRemaining - 1;
}
tw.Close();
}
ReportDone(false);
csv.Close();
}
/// <summary>
/// Process the file and output to the target file.
/// </summary>
/// <param name="target">The target file to write to.</param>
public void Process(string target)
{
var csv = new ReadCSV(InputFilename.ToString(), ExpectInputHeaders, Format);
TextWriter tw = new StreamWriter(target);
ResetStatus();
while (csv.Next())
{
var line = new StringBuilder();
UpdateStatus(false);
line.Append(GetColumnData(FileData.Date, csv));
line.Append(" ");
line.Append(GetColumnData(FileData.Time, csv));
line.Append(";");
line.Append(Format.Format(double.Parse(GetColumnData(FileData.Open, csv)), Precision));
line.Append(";");
line.Append(Format.Format(double.Parse(GetColumnData(FileData.High, csv)), Precision));
line.Append(";");
line.Append(Format.Format(double.Parse(GetColumnData(FileData.Low, csv)), Precision));
line.Append(";");
line.Append(Format.Format(double.Parse(GetColumnData(FileData.Close, csv)), Precision));
line.Append(";");
line.Append(Format.Format(double.Parse(GetColumnData(FileData.Volume, csv)), Precision));
tw.WriteLine(line.ToString());
}
ReportDone(false);
csv.Close();
tw.Close();
}
/// <summary>
/// Process the file.
/// </summary>
/// <param name="outputFile">The output file.</param>
/// <param name="method">The method to use.</param>
public void Process(FileInfo outputFile, IMLRegression method)
{
var csv = new ReadCSV(InputFilename.ToString(),
ExpectInputHeaders, Format);
if (method.InputCount != _inputCount)
{
throw new AnalystError("This machine learning method has "
+ method.InputCount
+ " inputs, however, the data has " + _inputCount
+ " inputs.");
}
var input = new BasicMLData(method.InputCount);
StreamWriter tw = AnalystPrepareOutputFile(outputFile);
ResetStatus();
while (csv.Next())
{
UpdateStatus(false);
var row = new LoadedRow(csv, _idealCount);
int dataIndex = 0;
// load the input data
for (int i = 0; i < _inputCount; i++)
{
String str = row.Data[i];
double d = Format.Parse(str);
input[i] = d;
dataIndex++;
}
// do we need to skip the ideal values?
dataIndex += _idealCount;
// compute the result
IMLData output = method.Compute(input);
// display the computed result
for (int i = 0; i < _outputCount; i++)
{
double d = output[i];
row.Data[dataIndex++] = Format.Format(d, Precision);
}
WriteRow(tw, row);
}
ReportDone(false);
tw.Close();
csv.Close();
}
/// <summary>
/// Process, and generate the output file.
/// </summary>
/// <param name="outputFile">The output file.</param>
public void Process(FileInfo outputFile)
{
ValidateAnalyzed();
var csv = new ReadCSV(InputFilename.ToString(),
ExpectInputHeaders, Format);
LoadedRow row;
StreamWriter tw = PrepareOutputFile(outputFile);
ResetStatus();
while ((row = GetNextRow(csv)) != null)
{
WriteRow(tw, row);
UpdateStatus(false);
}
ReportDone(false);
tw.Close();
csv.Close();
}
/// <summary>
/// Read the CSV file.
/// </summary>
private void ReadFile()
{
ReadCSV csv = null;
try
{
csv = new ReadCSV(InputFilename.ToString(),
ExpectInputHeaders, Format);
ResetStatus();
int row = 0;
while (csv.Next() && !ShouldStop())
{
UpdateStatus("Reading data");
foreach (BaseCachedColumn column in Columns)
{
if (column is FileData)
{
if (column.Input)
{
var fd = (FileData)column;
String str = csv.Get(fd.Index);
double d = Format.Parse(str);
fd.Data[row] = d;
}
}
}
row++;
}
}
finally
{
ReportDone("Reading data");
if (csv != null)
{
csv.Close();
}
}
}
/// <summary>
/// Process and balance the data.
/// </summary>
/// <param name="outputFile">The output file to write data to.</param>
/// <param name="targetField"></param>
/// <param name="countPer">The desired count per class.</param>
public void Process(FileInfo outputFile, int targetField,
int countPer)
{
ValidateAnalyzed();
StreamWriter tw = PrepareOutputFile(outputFile);
_counts = new Dictionary <String, Int32>();
var csv = new ReadCSV(InputFilename.ToString(),
ExpectInputHeaders, Format);
ResetStatus();
while (csv.Next() && !ShouldStop())
{
var row = new LoadedRow(csv);
UpdateStatus(false);
String key = row.Data[targetField];
int count;
if (!_counts.ContainsKey(key))
{
count = 0;
}
else
{
count = _counts[key];
}
if (count < countPer)
{
WriteRow(tw, row);
count++;
}
_counts[key] = count;
}
ReportDone(false);
csv.Close();
tw.Close();
}
/// <summary>
/// Process the input file.
/// </summary>
/// <param name="outputFile">The output file to write to.</param>
public void Process(FileInfo outputFile)
{
var csv = new ReadCSV(InputFilename.ToString(),
ExpectInputHeaders, Format);
StreamWriter tw = PrepareOutputFile(outputFile);
_filteredCount = 0;
ResetStatus();
while (csv.Next() && !ShouldStop())
{
UpdateStatus(false);
var row = new LoadedRow(csv);
if (ShouldProcess(row))
{
WriteRow(tw, row);
_filteredCount++;
}
}
ReportDone(false);
tw.Close();
csv.Close();
}
/// <summary>
/// Normalize the input file. Write to the specified file.
/// </summary>
///
/// <param name="file">The file to write to.</param>
public void Normalize(FileInfo file)
{
if (_analyst == null)
{
throw new EncogError(
"Can't normalize yet, file has not been analyzed.");
}
ReadCSV csv = null;
StreamWriter tw = null;
try
{
csv = new ReadCSV(InputFilename.ToString(),
ExpectInputHeaders, InputFormat);
file.Delete();
tw = new StreamWriter(file.OpenWrite());
// write headers, if needed
if (ProduceOutputHeaders)
{
WriteHeaders(tw);
}
ResetStatus();
int outputLength = _analyst.DetermineTotalColumns();
// write file contents
while (csv.Next() && !ShouldStop())
{
UpdateStatus(false);
double[] output = ExtractFields(
_analyst, _analystHeaders, csv, outputLength,
false);
if (_series.TotalDepth > 1)
{
output = _series.Process(output);
}
if (output != null)
{
var line = new StringBuilder();
NumberList.ToList(OutputFormat, line, output);
tw.WriteLine(line);
}
}
}
catch (IOException e)
{
throw new QuantError(e);
}
finally
{
ReportDone(false);
if (csv != null)
{
try
{
csv.Close();
}
catch (Exception ex)
{
EncogLogging.Log(ex);
}
}
if (tw != null)
{
try
{
tw.Close();
}
catch (Exception ex)
{
EncogLogging.Log(ex);
}
}
}
}
/// <summary>
/// Process the file.
/// </summary>
///
/// <param name="outputFile">The output file.</param>
/// <param name="method">THe method to use.</param>
public void Process(FileInfo outputFile, IMLMethod method)
{
var csv = new ReadCSV(InputFilename.ToString(),
ExpectInputHeaders, Format);
IMLData output;
foreach (AnalystField field in _analyst.Script.Normalize.NormalizedFields)
{
field.Init();
}
int outputLength = _analyst.DetermineTotalInputFieldCount();
StreamWriter tw = PrepareOutputFile(outputFile);
ResetStatus();
while (csv.Next())
{
UpdateStatus(false);
var row = new LoadedRow(csv, _outputColumns);
double[] inputArray = AnalystNormalizeCSV.ExtractFields(_analyst,
_analystHeaders, csv, outputLength, true);
if (_series.TotalDepth > 1)
{
inputArray = _series.Process(inputArray);
}
if (inputArray != null)
{
IMLData input = new BasicMLData(inputArray);
// evaluation data
if ((method is IMLClassification) &&
!(method is IMLRegression))
{
// classification only?
var tmp = new BasicMLData(1);
tmp[0] = ((IMLClassification)method).Classify(input);
output = tmp;
}
else
{
// regression
output = ((IMLRegression)method).Compute(input);
}
// skip file data
int index = _fileColumns;
int outputIndex = 0;
// display output
foreach (AnalystField field in _analyst.Script.Normalize.NormalizedFields)
{
if (_analystHeaders.Find(field.Name) != -1)
{
if (field.Output)
{
if (field.Classify)
{
// classification
ClassItem cls = field.DetermineClass(
outputIndex, output);
outputIndex += field.ColumnsNeeded;
if (cls == null)
{
row.Data[index++] = "?Unknown?";
}
else
{
row.Data[index++] = cls.Name;
}
}
else
{
// regression
double n = output[outputIndex++];
n = field.DeNormalize(n);
row.Data[index++] = Format
.Format(n, Precision);
}
}
}
}
}
WriteRow(tw, row);
}
ReportDone(false);
tw.Close();
csv.Close();
}
