| public GetDouble ( String column ) : double | ||
| column | String | The column to read. |
| return | double | |
protected override void LoadTestData(string testFile)
{
ReadCSV test_csv = new ReadCSV(testFile, true, CSVFormat.DecimalPoint);
List<double[]> test_input = new List<double[]>();
test_input_orig = new List<double[]>();
while (test_csv.Next())
{
double x = test_csv.GetDouble(0);
double y = test_csv.GetDouble(1);
test_input.Add(new[] { x, y });
test_input_orig.Add(new[] { x, y });
}
test_csv.Close();
//Analyze(ref test_input);
Normalize(ref test_input, ref vmin, ref vmax);
testData = new List<IMLData>();
foreach (var d in test_input)
{
testData.Add(new BasicMLData(d));
}
}
/// <summary>
/// Load a CSV file into a memory dataset.
/// </summary>
///
/// <param name="format">The CSV format to use.</param>
/// <param name="filename">The filename to load.</param>
/// <param name="headers">True if there is a header line.</param>
/// <param name="inputSize">The input size. Input always comes first in a file.</param>
/// <param name="idealSize">The ideal size, 0 for unsupervised.</param>
/// <returns>A NeuralDataSet that holds the contents of the CSV file.</returns>
public static IMLDataSet LoadCSVTOMemory(CSVFormat format, String filename,
bool headers, int inputSize, int idealSize)
{
var result = new BasicMLDataSet();
var csv = new ReadCSV(filename, headers, format);
while (csv.Next())
{
BasicMLData ideal = null;
int index = 0;
var input = new BasicMLData(inputSize);
for (int i = 0; i < inputSize; i++)
{
double d = csv.GetDouble(index++);
input[i] = d;
}
if (idealSize > 0)
{
ideal = new BasicMLData(idealSize);
for (int i = 0; i < idealSize; i++)
{
double d = csv.GetDouble(index++);
ideal[i] = d;
}
}
IMLDataPair pair = new BasicMLDataPair(input, ideal);
result.Add(pair);
}
return result;
}
public ICollection<LoadedMarketData> ReadAndCallLoader(TickerSymbol symbol, IList<MarketDataType> neededTypes, DateTime from, DateTime to, string File)
{
try
{
//We got a file, lets load it.
ICollection<LoadedMarketData> result = new List<LoadedMarketData>();
ReadCSV csv = new ReadCSV(File, true, CSVFormat.English);
csv.DateFormat = "yyyy.MM.dd HH:mm:ss";
DateTime ParsedDate = from;
// Time,Open,High,Low,Close,Volume
while (csv.Next() && ParsedDate >= from && ParsedDate <= to )
{
DateTime date = csv.GetDate("Time");
double Bid= csv.GetDouble("Bid");
double Ask = csv.GetDouble("Ask");
double AskVolume = csv.GetDouble("AskVolume");
double BidVolume= csv.GetDouble("BidVolume");
double _trade = ( Bid + Ask ) /2;
double _tradeSize = (AskVolume + BidVolume) / 2;
LoadedMarketData data = new LoadedMarketData(date, symbol);
data.SetData(MarketDataType.Trade, _trade);
data.SetData(MarketDataType.Volume, _tradeSize);
result.Add(data);
Console.WriteLine("Current DateTime:"+ParsedDate.ToShortDateString()+ " Time:"+ParsedDate.ToShortTimeString() +" Start date was "+from.ToShortDateString());
Console.WriteLine("Stopping at date:" + to.ToShortDateString() );
ParsedDate = date;
//double open = csv.GetDouble("Open");
//double close = csv.GetDouble("High");
//double high = csv.GetDouble("Low");
//double low = csv.GetDouble("Close");
//double volume = csv.GetDouble("Volume");
//LoadedMarketData data = new LoadedMarketData(date, symbol);
//data.SetData(MarketDataType.Open, open);
//data.SetData(MarketDataType.High, high);
//data.SetData(MarketDataType.Low, low);
//data.SetData(MarketDataType.Close, close);
//data.SetData(MarketDataType.Volume, volume);
result.Add(data);
}
csv.Close();
return result;
}
catch (Exception ex)
{
Console.WriteLine("Something went wrong reading the csv");
Console.WriteLine("Something went wrong reading the csv:" + ex.Message);
}
Console.WriteLine("Something went wrong reading the csv");
return null;
}
/// <summary>
/// Reads the CSV and call loader.
/// Used internally to load the csv and place data in the marketdataset.
/// </summary>
/// <param name="symbol">The symbol.</param>
/// <param name="neededTypes">The needed types.</param>
/// <param name="from">From.</param>
/// <param name="to">To.</param>
/// <param name="File">The file.</param>
/// <returns></returns>
ICollection<LoadedMarketData> ReadAndCallLoader(TickerSymbol symbol, IEnumerable<MarketDataType> neededTypes, DateTime from, DateTime to, string File)
{
//We got a file, lets load it.
ICollection<LoadedMarketData> result = new List<LoadedMarketData>();
ReadCSV csv = new ReadCSV(File, true, CSVFormat.English);
//In case we want to use a different date format...and have used the SetDateFormat method, our DateFormat must then not be null..
//We will use the ?? operator to check for nullables.
csv.DateFormat = DateFormat ?? "yyyy-MM-dd HH:mm:ss";
csv.TimeFormat = "HH:mm:ss";
DateTime ParsedDate = from;
bool writeonce = true;
while (csv.Next())
{
DateTime date = csv.GetDate(0);
ParsedDate = date;
if (writeonce)
{
Console.WriteLine(@"First parsed date in csv:" + ParsedDate.ToShortDateString());
Console.WriteLine(@"Stopping at date:" + to.ToShortDateString());
Console.WriteLine(@"Current DateTime:" + ParsedDate.ToShortDateString() + @" Time:" +
ParsedDate.ToShortTimeString() + @" Asked Start date was " +
from.ToShortDateString());
writeonce = false;
}
if (ParsedDate >= from && ParsedDate <= to)
{
DateTime datex = csv.GetDate(0);
double open = csv.GetDouble(1);
double close = csv.GetDouble(2);
double high = csv.GetDouble(3);
double low = csv.GetDouble(4);
double volume = csv.GetDouble(5);
double range = Math.Abs(open - close);
double HighLowRange = Math.Abs(high - low);
double DirectionalRange = close - open;
LoadedMarketData data = new LoadedMarketData(datex, symbol);
data.SetData(MarketDataType.Open, open);
data.SetData(MarketDataType.High, high);
data.SetData(MarketDataType.Low, low);
data.SetData(MarketDataType.Close, close);
data.SetData(MarketDataType.Volume, volume);
data.SetData(MarketDataType.RangeHighLow, Math.Round(HighLowRange, 6));
data.SetData(MarketDataType.RangeOpenClose, Math.Round(range, 6));
data.SetData(MarketDataType.RangeOpenCloseNonAbsolute, Math.Round(DirectionalRange, 6));
result.Add(data);
}
}
csv.Close();
return result;
}
public ICollection<LoadedMarketData> ReadAndCallLoader(TickerSymbol symbol, IList<MarketDataType> neededTypes, DateTime from, DateTime to,string File)
{
try
{
//We got a file, lets load it.
ICollection<LoadedMarketData> result = new List<LoadedMarketData>();
ReadCSV csv = new ReadCSV(File, true,LoadedFormat);
csv.DateFormat = DateTimeFormat.Normalize();
// Time,Open,High,Low,Close,Volume
while (csv.Next())
{
DateTime date = csv.GetDate("Time");
double open = csv.GetDouble("Open");
double close = csv.GetDouble("High");
double high = csv.GetDouble("Low");
double low = csv.GetDouble("Close");
double volume = csv.GetDouble("Volume");
LoadedMarketData data = new LoadedMarketData(date, symbol);
data.SetData(MarketDataType.Open, open);
data.SetData(MarketDataType.High, high);
data.SetData(MarketDataType.Low, low);
data.SetData(MarketDataType.Close, close);
data.SetData(MarketDataType.Volume, volume);
result.Add(data);
}
csv.Close();
return result;
}
catch (Exception ex)
{
Console.WriteLine("Something went wrong reading the csv");
Console.WriteLine("Something went wrong reading the csv:"+ex.Message);
}
Console.WriteLine("Something went wrong reading the csv");
return null;
}
/// <summary>
/// parses one column of a csv and returns an array of doubles.
/// you can only return one double array with this method.
/// </summary>
/// <param name="file">The file.</param>
/// <param name="formatused">The formatused.</param>
/// <param name="Name">The name of the column to parse..</param>
/// <returns></returns>
public static List<double> QuickParseCSV(string file, CSVFormat formatused, string Name)
{
List<double> returnedArrays = new List<double>();
ReadCSV csv = new ReadCSV(file, true, formatused);
while (csv.Next())
{
returnedArrays.Add(csv.GetDouble(Name));
}
return returnedArrays;
}
/// <summary>
/// Load financial data from a CSV file.
/// </summary>
/// <param name="ticker">The ticker being loaded, ignored for a CSV load.</param>
/// <param name="dataNeeded">The data needed.</param>
/// <param name="from">The starting date.</param>
/// <param name="to">The ending date.</param>
/// <returns></returns>
public ICollection<LoadedMarketData> Load(TickerSymbol ticker, IList<MarketDataType> dataNeeded, DateTime from,
DateTime to)
{
try
{
if (File.Exists(TheFile))
{
//We got a file, lets load it.
TheFile = TheFile;
ICollection<LoadedMarketData> result = new List<LoadedMarketData>();
var csv = new ReadCSV(TheFile, true, CSVFormat.English);
// Time,Open,High,Low,Close,Volume
while (csv.Next())
{
DateTime date = csv.GetDate("Time");
double open = csv.GetDouble("Open");
double close = csv.GetDouble("High");
double high = csv.GetDouble("Low");
double low = csv.GetDouble("Close");
double volume = csv.GetDouble("Volume");
var data = new LoadedMarketData(date, ticker);
data.SetData(MarketDataType.Open, open);
data.SetData(MarketDataType.Volume, close);
data.SetData(MarketDataType.High, high);
data.SetData(MarketDataType.Low, low);
data.SetData(MarketDataType.Volume, volume);
result.Add(data);
}
csv.Close();
return result;
}
}
catch (Exception ex)
{
throw new LoaderError(ex);
}
throw new LoaderError(@"Something went wrong reading the csv");
}
/// <summary>
/// parses one column of a csv and returns an array of doubles.
/// you can only return one double array with this method.
/// We are assuming CSVFormat english in this quick parse csv method.
/// You can input the size (number of lines) to read.
/// </summary>
/// <param name="file">The file.</param>
/// <param name="Name">The name of the column to parse.</param>
/// <param name="size">The size.</param>
/// <returns></returns>
public static List<double> QuickParseCSV(string file, string Name, int size)
{
List<double> returnedArrays = new List<double>();
ReadCSV csv = new ReadCSV(file, true, CSVFormat.English);
int currentRead = 0;
while (csv.Next() && currentRead < size)
{
returnedArrays.Add(csv.GetDouble(Name));
currentRead++;
}
return returnedArrays;
}
/// <summary>
/// Load financial data from Google.
/// </summary>
/// <param name="ticker">The ticker to load from.</param>
/// <param name="dataNeeded">The data needed.</param>
/// <param name="from">The starting time.</param>
/// <param name="to">The ending time.</param>
/// <returns>The loaded data.</returns>
public ICollection<LoadedMarketData> Load(TickerSymbol ticker, IList<MarketDataType> dataNeeded, DateTime from,
DateTime to)
{
ICollection<LoadedMarketData> result = new List<LoadedMarketData>();
Uri url = BuildUrl(ticker, from, to);
WebRequest http = WebRequest.Create(url);
var response = (HttpWebResponse) http.GetResponse();
if (response != null)
using (Stream istream = response.GetResponseStream())
{
var csv = new ReadCSV(istream, true, CSVFormat.DecimalPoint);
while (csv.Next())
{
DateTime date = csv.GetDate("date");
double open = csv.GetDouble("open");
double close = csv.GetDouble("close");
double high = csv.GetDouble("high");
double low = csv.GetDouble("low");
double volume = csv.GetDouble("volume");
var data =
new LoadedMarketData(date, ticker);
data.SetData(MarketDataType.Open, open);
data.SetData(MarketDataType.Close, close);
data.SetData(MarketDataType.High, high);
data.SetData(MarketDataType.Low, low);
data.SetData(MarketDataType.Open, open);
data.SetData(MarketDataType.Volume, volume);
result.Add(data);
}
csv.Close();
if (istream != null) istream.Close();
}
return result;
}
/// <summary>
/// Convert a CSV file to binary.
/// </summary>
/// <param name="csvFile">The CSV file to convert.</param>
/// <param name="format">The format.</param>
/// <param name="binFile">The binary file.</param>
/// <param name="input">The input.</param>
/// <param name="ideal">The ideal.</param>
/// <param name="headers">True, if headers are present.</param>
public static void ConvertCSV2Binary(FileInfo csvFile, CSVFormat format,
FileInfo binFile, int[] input, int[] ideal, bool headers)
{
binFile.Delete();
var csv = new ReadCSV(csvFile.ToString(), headers, format);
var buffer = new BufferedMLDataSet(binFile.ToString());
buffer.BeginLoad(input.Length, ideal.Length);
while (csv.Next())
{
var inputData = new BasicMLData(input.Length);
var idealData = new BasicMLData(ideal.Length);
// handle input data
for (int i = 0; i < input.Length; i++)
{
inputData[i] = csv.GetDouble(input[i]);
}
// handle input data
for (int i = 0; i < ideal.Length; i++)
{
idealData[i] = csv.GetDouble(ideal[i]);
}
// add to dataset
buffer.Add(inputData, idealData);
}
buffer.EndLoad();
}
/// <summary>
/// Used to calibrate the training file.
/// </summary>
/// <param name="file">The file to consider.</param>
protected void CalibrateFile(string file)
{
var csv = new ReadCSV(file, true, CSVFormat.English);
while (csv.Next())
{
var a = new double[1];
double close = csv.GetDouble(1);
const int fastIndex = 2;
const int slowIndex = fastIndex + Config.InputWindow;
a[0] = close;
for (int i = 0; i < Config.InputWindow; i++)
{
double fast = csv.GetDouble(fastIndex + i);
double slow = csv.GetDouble(slowIndex + i);
if (!double.IsNaN(fast) && !double.IsNaN(slow))
{
double diff = (fast - slow)/Config.PipSize;
_minDifference = Math.Min(_minDifference, diff);
_maxDifference = Math.Max(_maxDifference, diff);
}
}
_window.Add(a);
if (_window.IsFull())
{
double max = (_window.CalculateMax(0, Config.InputWindow) - close)/Config.PipSize;
double min = (_window.CalculateMin(0, Config.InputWindow) - close)/Config.PipSize;
double o = Math.Abs(max) > Math.Abs(min) ? max : min;
_maxPiPs = Math.Max(_maxPiPs, (int) o);
_minPiPs = Math.Min(_minPiPs, (int) o);
}
}
}
/// <summary>
/// Process the individual training file.
/// </summary>
/// <param name="file">The training file to process.</param>
/// <param name="output">The data set to output to.</param>
protected void ProcessFile(string file, BufferedMLDataSet output)
{
var inputData = new BasicMLData(output.InputSize);
var idealData = new BasicMLData(output.IdealSize);
var csv = new ReadCSV(file, true, CSVFormat.English);
while (csv.Next())
{
var a = new double[Config.InputWindow + 1];
double close = csv.GetDouble(1);
const int fastIndex = 2;
const int slowIndex = fastIndex + Config.InputWindow;
a[0] = close;
for (int i = 0; i < 3; i++)
{
double fast = csv.GetDouble(fastIndex + i);
double slow = csv.GetDouble(slowIndex + i);
double diff = _fieldDifference.Normalize((fast - slow)/Config.PipSize);
a[i + 1] = diff;
}
_window.Add(a);
if (_window.IsFull())
{
double max = (_window.CalculateMax(0, Config.InputWindow) - close)/Config.PipSize;
double min = (_window.CalculateMin(0, Config.InputWindow) - close)/Config.PipSize;
double o = Math.Abs(max) > Math.Abs(min) ? max : min;
a = _window.GetLast();
for (int i = 0; i < 3; i++)
{
inputData[i] = a[i + 1];
}
o = _fieldOutcome.Normalize(o);
idealData[0] = o;
output.Add(inputData, idealData);
}
}
}
/// <summary>
/// Reads and parses CSV data from file
/// </summary>
/// <param name="ticker">Ticker associated with CSV file</param>
/// <param name="neededTypes">Columns to parse (headers)</param>
/// <param name="from">DateTime from</param>
/// <param name="to">DateTime to</param>
/// <param name="File">Filepath to CSV</param>
/// <returns>Marketdata</returns>
public ICollection<LoadedMarketData> ReadAndCallLoader(TickerSymbol ticker, IList<MarketDataType> neededTypes, DateTime from, DateTime to, string File)
{
try
{
LoadedFile = File;
Console.WriteLine("Loading instrument: " + ticker.Symbol + " from: " + File);
//We got a file, lets load it.
ICollection<LoadedMarketData> result = new List<LoadedMarketData>();
ReadCSV csv = new ReadCSV(File, true, LoadedFormat);
if (DateTimeDualColumn)
{
csv.DateFormat = DateFormat;
csv.TimeFormat = TimeFormat;
}
else
{
csv.DateFormat = DateFormat;
}
//"Date","Time","Open","High","Low","Close","Volume"
while (csv.Next())
{
string datetime = "";
if (DateTimeDualColumn)
{
datetime = csv.GetDate("Date").ToShortDateString() + " " +
csv.GetTime("Time").ToShortTimeString();
}
else
{
datetime = csv.GetDate("Date").ToShortDateString();
}
DateTime date = DateTime.Parse(datetime);
if (date > from && date < to)
{
// CSV columns
double open = csv.GetDouble("Open");
double high = csv.GetDouble("High");
double low = csv.GetDouble("Low");
double close = csv.GetDouble("Close");
double volume = csv.GetDouble("Volume");
LoadedMarketData data = new LoadedMarketData(date, ticker);
foreach (MarketDataType marketDataType in neededTypes)
{
switch (marketDataType.ToString())
{
case "Open":
data.SetData(MarketDataType.Open, open);
break;
case "High":
data.SetData(MarketDataType.High, high);
break;
case "Low":
data.SetData(MarketDataType.Low, low);
break;
case "Close":
data.SetData(MarketDataType.Close, close);
break;
case "Volume":
data.SetData(MarketDataType.Volume, volume);
break;
case "RangeHighLow":
data.SetData(MarketDataType.RangeHighLow, Math.Round(Math.Abs(high - low), 6));
break;
case "RangeOpenClose":
data.SetData(MarketDataType.RangeOpenClose, Math.Round(Math.Abs(close - open), 6));
break;
case "RangeOpenCloseNonAbsolute":
data.SetData(MarketDataType.RangeOpenCloseNonAbsolute, Math.Round(close - open, 6));
break;
case "Weighted":
data.SetData(MarketDataType.Weighted, Math.Round((high + low + 2 * close) / 4, 6));
break;
}
}
result.Add(data);
}
}
csv.Close();
return result;
}
catch (Exception ex)
{
Console.WriteLine("Something went wrong reading the csv: " + ex.Message);
}
return null;
}
/// <summary>
/// parses one column of a csv and returns an array of doubles.
/// you can only return one double array with this method.
/// We are assuming CSVFormat english in this quick parse csv method.
/// You can input the size (number of lines) to read , and the number of lines you want to skip start from the first line.
/// </summary>
/// <param name="file">The file.</param>
/// <param name="columnNumber">The column number to get.</param>
/// <param name="size">The size.</param>
/// <param name="startLine">The start line (how many lines you want to skip from the first line.</param>
/// <returns></returns>
public static List<double> QuickParseCSV(string file, int columnNumber, int size, int startLine)
{
List<double> returnedArrays = new List<double>();
ReadCSV csv = new ReadCSV(file, true, CSVFormat.English);
int currentRead = 0;
int currentLine = 0;
while (csv.Next())
{
if (currentRead < size && currentLine > startLine)
{
returnedArrays.Add(csv.GetDouble(columnNumber));
currentRead++;
}
currentLine++;
}
return returnedArrays;
}
public void loadSP500(String sp500Filename)
{
var csv = new ReadCSV(sp500Filename, true, CSVFormat.English);
while (csv.Next())
{
var date = csv.GetDate("date");
double amount = csv.GetDouble("adj close");
var sample = new FinancialSample();
sample.setAmount(amount);
sample.setDate(date);
samples.Add(sample);
}
csv.Close();
samples.Sort();
}
public void loadPrime(String primeFilename)
{
var csv = new ReadCSV(primeFilename, true, CSVFormat.English);
while (csv.Next())
{
var date = csv.GetDate("date");
double rate = csv.GetDouble("prime");
var ir = new InterestRate(date, rate);
rates.Add(ir);
}
csv.Close();
rates.Sort();
}
public void TestModel(string testDataPath, BasicNetwork model, ProblemType problem, ActivationType activation)
{
TestDataPath = testDataPath;
var csvReader = new ReadCSV(testDataPath, true, CSVFormat.DecimalPoint);
var values = new List<double[]>();
var originalValues = new List<double[]>();
while (csvReader.Next())
{
values.Add(ProblemType.Classification == problem
? new[] {csvReader.GetDouble(0), csvReader.GetDouble(1)}
: new[] {csvReader.GetDouble(0)});
originalValues.Add(ProblemType.Classification == problem
? new[] { csvReader.GetDouble(0), csvReader.GetDouble(1) }
: new[] { csvReader.GetDouble(0) });
}
csvReader.Close();
Normalize(values, _valuesMins, _valuesMaxes, activation);
var answers = new List<double>();
foreach (var value in values)
{
var answer = new double[LastLayerSize];
model.Compute(value, answer);
answers.Add(problem == ProblemType.Regression ? DenormalizeAnswer(answer[0], activation) : GetClassFromAnswer(answer));
}
AnswerPath = Path.GetFullPath(TestDataPath) + ".solved";
var lines = new List<string>();
lines.Add(problem == ProblemType.Classification ? "x,y,clc" : "x,y");
lines.AddRange(answers.Select((t, i) =>
problem == ProblemType.Regression
? originalValues[i][0].ToString(CultureInfo.InvariantCulture) + "," + t.ToString(CultureInfo.InvariantCulture)
: originalValues[i][0].ToString(CultureInfo.InvariantCulture) + "," + originalValues[i][1].ToString(CultureInfo.InvariantCulture) + "," + t.ToString(CultureInfo.InvariantCulture)));
File.WriteAllLines(AnswerPath, lines);
}
public void LoadTrainingData(string trainingDataPath, ProblemType problem, ActivationType activation)
{
TrainingDataPath = trainingDataPath;
var csvReader = new ReadCSV(trainingDataPath, true, CSVFormat.DecimalPoint);
var values = new List<double[]>();
var answers = new List<double[]>();
while (csvReader.Next())
{
if (ProblemType.Classification == problem)
{
values.Add(new []{csvReader.GetDouble(0), csvReader.GetDouble(1)});
answers.Add(new []{csvReader.GetDouble(2)});
}
else
{
values.Add(new[] { csvReader.GetDouble(0)});
answers.Add(new[] { csvReader.GetDouble(1) });
_originalRegressionValues.Add(values.Last()[0]);
_originalRegressionAnswers.Add(answers.Last()[0]);
}
}
csvReader.Close();
if (problem == ProblemType.Classification)
{
answers = SpreadClassificationAnswers(answers, activation);
FirstLayerSize = 2;
}
else
LastLayerSize = FirstLayerSize = 1;
AnalizeValues(problem, values);
Normalize(values, _valuesMins, _valuesMaxes, activation);
if (problem == ProblemType.Regression)
{
AnalizeAnswers(answers);
Normalize(answers, _answersMins, _answersMaxes, activation);
}
values.StableShuffle();
answers.StableShuffle();
ListExtensions.ResetStableShuffle();
var trainingSetSize = (int)(values.Count * 0.85);
TrainingDataSet = new BasicMLDataSet(values.Take(trainingSetSize).ToArray(), answers.Take(trainingSetSize).ToArray());
ValidationDataSet = new BasicMLDataSet(values.Skip(trainingSetSize).ToArray(), answers.Skip(trainingSetSize).ToArray());
}
protected override void LoadTrainData(string trainingFile)
{
ReadCSV train_csv = new ReadCSV(trainingFile, true, CSVFormat.DecimalPoint);
List<double[]> train_input = new List<double[]>();
List<double[]> train_ideal = new List<double[]>();
train_input_orig = new List<Results>();
while (train_csv.Next())
{
double x = train_csv.GetDouble(0);
double y = train_csv.GetDouble(1);
train_input.Add(new double[] { x });
train_ideal.Add(new double[] { y });
train_input_orig.Add(new Results(x, y, 0));
}
train_csv.Close();
Analyze(ref train_input, ref vmin, ref vmax);
Normalize(ref train_input, ref vmin, ref vmax);
Analyze(ref train_ideal, ref vmino, ref vmaxo);
Normalize(ref train_ideal, ref vmino, ref vmaxo);
Randomize(ref train_input, ref train_ideal);
int validation_size = train_input.Count / 10;
validationData = new BasicMLDataSet(train_input.Take(validation_size).ToArray(), train_ideal.Take(validation_size).ToArray());
trainingData = new BasicMLDataSet(train_input.Skip(validation_size).ToArray(), train_ideal.Skip(validation_size).ToArray());
}