/// <summary>
/// Convert a CSV file to binary.
/// </summary>
/// <param name="csvFile">The CSV file to convert.</param>
/// <param name="format">The format.</param>
/// <param name="binFile">The binary file.</param>
/// <param name="input">The input.</param>
/// <param name="ideal">The ideal.</param>
/// <param name="headers">True, if headers are present.</param>
public static void ConvertCSV2Binary(FileInfo csvFile, CSVFormat format,
FileInfo binFile, int[] input, int[] ideal, bool headers)
{
binFile.Delete();
var csv = new ReadCSV(csvFile.ToString(), headers, format);
var buffer = new BufferedMLDataSet(binFile.ToString());
buffer.BeginLoad(input.Length, ideal.Length);
while (csv.Next())
{
var inputData = new BasicMLData(input.Length);
var idealData = new BasicMLData(ideal.Length);
// handle input data
for (int i = 0; i < input.Length; i++)
{
inputData[i] = csv.GetDouble(input[i]);
}
// handle input data
for (int i = 0; i < ideal.Length; i++)
{
idealData[i] = csv.GetDouble(ideal[i]);
}
// add to dataset
buffer.Add(inputData, idealData);
}
buffer.EndLoad();
buffer.Close();
csv.Close();
}
/// <summary>
/// Convert a CSV file to a binary training file.
/// </summary>
/// <param name="csvFile">The CSV file.</param>
/// <param name="format">The format.</param>
/// <param name="binFile">The binary file.</param>
/// <param name="inputCount">The number of input values.</param>
/// <param name="outputCount">The number of output values.</param>
/// <param name="headers">True, if there are headers on the3 CSV.</param>
/// <param name="expectSignificance">Should a significance column be expected.</param>
public static void ConvertCSV2Binary(String csvFile, CSVFormat format,
String binFile, int inputCount, int outputCount,
bool headers, bool expectSignificance)
{
new FileInfo(binFile).Delete();
var csv = new CSVMLDataSet(csvFile,
inputCount, outputCount, false, format, expectSignificance);
var buffer = new BufferedMLDataSet(binFile);
buffer.BeginLoad(inputCount, outputCount);
foreach (IMLDataPair pair in csv)
{
buffer.Add(pair);
}
buffer.EndLoad();
}
/// <summary>
/// Process the individual training file.
/// </summary>
/// <param name="file">The training file to process.</param>
/// <param name="output">The data set to output to.</param>
protected void ProcessFile(string file, BufferedMLDataSet output)
{
var inputData = new BasicMLData(output.InputSize);
var idealData = new BasicMLData(output.IdealSize);
var csv = new ReadCSV(file, true, CSVFormat.English);
while (csv.Next())
{
var a = new double[Config.InputWindow + 1];
double close = csv.GetDouble(1);
const int fastIndex = 2;
const int slowIndex = fastIndex + Config.InputWindow;
a[0] = close;
for (int i = 0; i < 3; i++)
{
double fast = csv.GetDouble(fastIndex + i);
double slow = csv.GetDouble(slowIndex + i);
double diff = _fieldDifference.Normalize((fast - slow) / Config.PipSize);
a[i + 1] = diff;
}
_window.Add(a);
if (_window.IsFull())
{
double max = (_window.CalculateMax(0, Config.InputWindow) - close) / Config.PipSize;
double min = (_window.CalculateMin(0, Config.InputWindow) - close) / Config.PipSize;
double o = Math.Abs(max) > Math.Abs(min) ? max : min;
a = _window.GetLast();
for (int i = 0; i < 3; i++)
{
inputData[i] = a[i + 1];
}
o = _fieldOutcome.Normalize(o);
idealData[0] = o;
output.Add(inputData, idealData);
}
}
}