public void TestRegression()
{
FileInfo rawFile = TEMP_DIR.CreateFile("simple.csv");
FileInfo egaFile = TEMP_DIR.CreateFile("simple.ega");
FileInfo outputFile = TEMP_DIR.CreateFile("simple_output.csv");
FileUtil.CopyResource("Encog.Resources.simple.csv", rawFile);
FileUtil.CopyResource("Encog.Resources.simple-r.ega", egaFile);
EncogAnalyst analyst = new EncogAnalyst();
analyst.Load(egaFile);
analyst.ExecuteTask("task-full");
ReadCSV csv = new ReadCSV(outputFile.ToString(), true, CSVFormat.English);
while (csv.Next())
{
double diff = Math.Abs(csv.GetDouble(2) - csv.GetDouble(4));
Assert.IsTrue(diff < 1.5);
}
Assert.AreEqual(4, analyst.Script.Fields.Length);
Assert.AreEqual(3, analyst.Script.Fields[3].ClassMembers.Count);
csv.Close();
}
/// <summary>
/// Convert a CSV file to binary.
/// </summary>
/// <param name="csvFile">The CSV file to convert.</param>
/// <param name="format">The format.</param>
/// <param name="binFile">The binary file.</param>
/// <param name="input">The input.</param>
/// <param name="ideal">The ideal.</param>
/// <param name="headers">True, if headers are present.</param>
public static void ConvertCSV2Binary(FileInfo csvFile, CSVFormat format,
FileInfo binFile, int[] input, int[] ideal, bool headers)
{
binFile.Delete();
var csv = new ReadCSV(csvFile.ToString(), headers, format);
var buffer = new BufferedMLDataSet(binFile.ToString());
buffer.BeginLoad(input.Length, ideal.Length);
while (csv.Next())
{
var inputData = new BasicMLData(input.Length);
var idealData = new BasicMLData(ideal.Length);
// handle input data
for (int i = 0; i < input.Length; i++)
{
inputData[i] = csv.GetDouble(input[i]);
}
// handle input data
for (int i = 0; i < ideal.Length; i++)
{
idealData[i] = csv.GetDouble(ideal[i]);
}
// add to dataset
buffer.Add(inputData, idealData);
}
buffer.EndLoad();
buffer.Close();
csv.Close();
}
/// <summary>
/// Load a CSV file into a memory dataset.
/// </summary>
///
/// <param name="format">The CSV format to use.</param>
/// <param name="filename">The filename to load.</param>
/// <param name="headers">True if there is a header line.</param>
/// <param name="inputSize">The input size. Input always comes first in a file.</param>
/// <param name="idealSize">The ideal size, 0 for unsupervised.</param>
/// <returns>A NeuralDataSet that holds the contents of the CSV file.</returns>
public static IMLDataSet LoadCSVTOMemory(CSVFormat format, String filename,
bool headers, int inputSize, int idealSize)
{
var result = new BasicMLDataSet();
var csv = new ReadCSV(filename, headers, format);
while (csv.Next())
{
BasicMLData ideal = null;
int index = 0;
var input = new BasicMLData(inputSize);
for (int i = 0; i < inputSize; i++)
{
double d = csv.GetDouble(index++);
input[i] = d;
}
if (idealSize > 0)
{
ideal = new BasicMLData(idealSize);
for (int i = 0; i < idealSize; i++)
{
double d = csv.GetDouble(index++);
ideal[i] = d;
}
}
IMLDataPair pair = new BasicMLDataPair(input, ideal);
result.Add(pair);
}
return(result);
}
public void TestRegression()
{
FileInfo rawFile = TEMP_DIR.CreateFile("simple.csv");
FileInfo egaFile = TEMP_DIR.CreateFile("simple.ega");
FileInfo outputFile = TEMP_DIR.CreateFile("simple_output.csv");
FileUtil.CopyResource("Encog.Resources.simple.csv", rawFile);
FileUtil.CopyResource("Encog.Resources.simple-r.ega", egaFile);
EncogAnalyst analyst = new EncogAnalyst();
analyst.Load(egaFile);
analyst.ExecuteTask("task-full");
ReadCSV csv = new ReadCSV(outputFile.ToString(), true, CSVFormat.English);
while (csv.Next())
{
double diff = Math.Abs(csv.GetDouble(2) - csv.GetDouble(4));
Assert.IsTrue(diff < 1.5);
}
Assert.AreEqual(4, analyst.Script.Fields.Length);
Assert.AreEqual(3, analyst.Script.Fields[3].ClassMembers.Count);
csv.Close();
}
/// <inheritdoc/>
public bool Read(double[] input, double[] ideal, ref double significance)
{
if (_readCSV.Next())
{
int index = 0;
for (int i = 0; i < input.Length; i++)
{
input[i] = _readCSV.GetDouble(index++);
}
for (int i = 0; i < ideal.Length; i++)
{
ideal[i] = _readCSV.GetDouble(index++);
}
if (_significance)
{
significance = _readCSV.GetDouble(index++);
}
else
{
significance = 1;
}
return(true);
}
return(false);
}
public void LoadCSVFileTestClass()
{
var path = parameters.TestFile;
var csvRead = new ReadCSV(new FileStream(path, FileMode.Open), true, CSVFormat.DecimalPoint);
var values = new List <double[]>();
while (csvRead.Next())
{
values.Add(new double[2] {
csvRead.GetDouble(0), csvRead.GetDouble(1)
});
}
csvRead.Close();
for (int i = 0; i < values.Count; i++)
{
for (int argc = 0; argc < values[i].Length; argc++)
{
values[i][argc] = ArgNormalize[argc].Normalize(values[i][argc]);
}
}
//Normalization(values);
testSet = values.ToArray();
}
public void LoadCSVFileTrainingClass()
{
var path = parameters.TrainFile;
var csvRead = new ReadCSV(new FileStream(path, FileMode.Open), true, CSVFormat.DecimalPoint);
var values = new List <double[]>();
var classes = new List <double[]>();
while (csvRead.Next())
{
values.Add(new double[2] {
csvRead.GetDouble(0), csvRead.GetDouble(1)
});
classes.Add(new double[1] {
csvRead.GetDouble(2)
});
}
csvRead.Close();
var min = parameters.FunctionType == FunctionTypeEnum.Unipolar ? 0d : -1d;
ArgNormalize = new NormalizedField[2] {
new NormalizedField(NormalizationAction.Normalize, "X", values.Max(v => v[0]), values.Min(v => v[0]), 1.0, min),
new NormalizedField(NormalizationAction.Normalize, "Y", values.Max(v => v[1]), values.Min(v => v[1]), 1.0, min)
};
for (int i = 0; i < values.Count; i++)
{
for (int argc = 0; argc < values[i].Length; argc++)
{
values[i][argc] = ArgNormalize[argc].Normalize(values[i][argc]);
}
}
//Normalization(values);
classCount = classes.Select(c => c[0]).Distinct().Count();
var normalizeClasses = new List <double[]>();
for (int i = 0; i < classes.Count; ++i)
{
var newClasses = new double[classCount];
for (int j = 0; j < newClasses.Length; j++)
{
newClasses[j] = min;
}
newClasses[(int)classes[i][0] - 1] = 1;// dodoac normalizacje na -1
normalizeClasses.Add(newClasses);
}
var trainSetCount = (int)((double)values.Count * ((100.0 - 15) / 100));
values.Shuffle();
normalizeClasses.Shuffle();
MyExtensions.ResetStableShuffle();
TrainSet = new BasicNeuralDataSet(values.Take(trainSetCount).ToArray(),
normalizeClasses.Take(trainSetCount).ToArray());
ValidationSet = new BasicNeuralDataSet(values.Skip(trainSetCount).ToArray(),
normalizeClasses.Skip(trainSetCount).ToArray());
}
public ICollection <LoadedMarketData> ReadAndCallLoader(TickerSymbol symbol, IList <MarketDataType> neededTypes, DateTime from, DateTime to, string File)
{
try
{
//We got a file, lets load it.
ICollection <LoadedMarketData> result = new List <LoadedMarketData>();
ReadCSV csv = new ReadCSV(File, true, CSVFormat.English);
csv.DateFormat = "yyyy.MM.dd HH:mm:ss";
DateTime ParsedDate = from;
// Time,Open,High,Low,Close,Volume
while (csv.Next() && ParsedDate >= from && ParsedDate <= to)
{
DateTime date = csv.GetDate("Time");
double Bid = csv.GetDouble("Bid");
double Ask = csv.GetDouble("Ask");
double AskVolume = csv.GetDouble("AskVolume");
double BidVolume = csv.GetDouble("BidVolume");
double _trade = (Bid + Ask) / 2;
double _tradeSize = (AskVolume + BidVolume) / 2;
LoadedMarketData data = new LoadedMarketData(date, symbol);
data.SetData(MarketDataType.Trade, _trade);
data.SetData(MarketDataType.Volume, _tradeSize);
result.Add(data);
Console.WriteLine("Current DateTime:" + ParsedDate.ToShortDateString() + " Time:" + ParsedDate.ToShortTimeString() + " Start date was " + from.ToShortDateString());
Console.WriteLine("Stopping at date:" + to.ToShortDateString());
ParsedDate = date;
//double open = csv.GetDouble("Open");
//double close = csv.GetDouble("High");
//double high = csv.GetDouble("Low");
//double low = csv.GetDouble("Close");
//double volume = csv.GetDouble("Volume");
//LoadedMarketData data = new LoadedMarketData(date, symbol);
//data.SetData(MarketDataType.Open, open);
//data.SetData(MarketDataType.High, high);
//data.SetData(MarketDataType.Low, low);
//data.SetData(MarketDataType.Close, close);
//data.SetData(MarketDataType.Volume, volume);
result.Add(data);
}
csv.Close();
return(result);
}
catch (Exception ex)
{
Console.WriteLine("Something went wrong reading the csv");
Console.WriteLine("Something went wrong reading the csv:" + ex.Message);
}
Console.WriteLine("Something went wrong reading the csv");
return(null);
}
/// <summary>
/// Reads the CSV and call loader.
/// Used internally to load the csv and place data in the marketdataset.
/// </summary>
/// <param name="symbol">The symbol.</param>
/// <param name="neededTypes">The needed types.</param>
/// <param name="from">From.</param>
/// <param name="to">To.</param>
/// <param name="File">The file.</param>
/// <returns></returns>
ICollection <LoadedMarketData> ReadAndCallLoader(TickerSymbol symbol, IEnumerable <MarketDataType> neededTypes, DateTime from, DateTime to, string File)
{
//We got a file, lets load it.
ICollection <LoadedMarketData> result = new List <LoadedMarketData>();
ReadCSV csv = new ReadCSV(File, true, CSVFormat.English);
//In case we want to use a different date format...and have used the SetDateFormat method, our DateFormat must then not be null..
//We will use the ?? operator to check for nullables.
csv.DateFormat = DateFormat ?? "yyyy-MM-dd HH:mm:ss";
csv.TimeFormat = "HH:mm:ss";
DateTime ParsedDate = from;
bool writeonce = true;
while (csv.Next())
{
DateTime date = csv.GetDate(0);
ParsedDate = date;
if (writeonce)
{
Console.WriteLine(@"First parsed date in csv:" + ParsedDate.ToShortDateString());
Console.WriteLine(@"Stopping at date:" + to.ToShortDateString());
Console.WriteLine(@"Current DateTime:" + ParsedDate.ToShortDateString() + @" Time:" +
ParsedDate.ToShortTimeString() + @" Asked Start date was " +
from.ToShortDateString());
writeonce = false;
}
if (ParsedDate >= from && ParsedDate <= to)
{
DateTime datex = csv.GetDate(0);
double open = csv.GetDouble(1);
double close = csv.GetDouble(2);
double high = csv.GetDouble(3);
double low = csv.GetDouble(4);
double volume = csv.GetDouble(5);
double range = Math.Abs(open - close);
double HighLowRange = Math.Abs(high - low);
double DirectionalRange = close - open;
LoadedMarketData data = new LoadedMarketData(datex, symbol);
data.SetData(MarketDataType.Open, open);
data.SetData(MarketDataType.High, high);
data.SetData(MarketDataType.Low, low);
data.SetData(MarketDataType.Close, close);
data.SetData(MarketDataType.Volume, volume);
data.SetData(MarketDataType.RangeHighLow, Math.Round(HighLowRange, 6));
data.SetData(MarketDataType.RangeOpenClose, Math.Round(range, 6));
data.SetData(MarketDataType.RangeOpenCloseNonAbsolute, Math.Round(DirectionalRange, 6));
result.Add(data);
}
}
csv.Close();
return(result);
}
public ICollection <LoadedMarketData> Load(
TickerSymbol ticker,
IList <MarketDataType> dataNeeded,
DateTime from,
DateTime to)
{
// TODO: nyyyyyyyaaagh!
ICollection <LoadedMarketData> result =
new List <LoadedMarketData>();
Uri url = BuildURL(ticker, from, to);
WebRequest http = HttpWebRequest.Create(url);
HttpWebResponse response = http.GetResponse() as HttpWebResponse;
using (Stream istream = response.GetResponseStream())
{
ReadCSV csv = new ReadCSV(
istream,
true,
CSVFormat.DECIMAL_POINT
);
while (csv.Next())
{
// todo: edit headers to match
DateTime date = csv.GetDate("DATE");
date =
date.Add(
new TimeSpan(
csv.GetDate("TIME").Hour,
csv.GetDate("TIME").Minute,
csv.GetDate("TIME").Second
)
);
double open = csv.GetDouble("OPEN");
double high = csv.GetDouble("MIN");
double low = csv.GetDouble("MAX");
double close = csv.GetDouble("CLOSE");
double volume = csv.GetDouble("VOLUME");
LoadedMarketData data =
new LoadedMarketData(date, ticker);
data.SetData(MarketDataType.OPEN, open);
data.SetData(MarketDataType.HIGH, high);
data.SetData(MarketDataType.LOW, low);
data.SetData(MarketDataType.CLOSE, close);
data.SetData(MarketDataType.VOLUME, volume);
result.Add(data);
}
csv.Close();
istream.Close();
}
return(result);
}
/// <summary>
/// Called internally to obtain the current value for an input field.
/// </summary>
/// <param name="field">The input field to determine.</param>
/// <param name="index">The current index.</param>
/// <returns>The value for this input field.</returns>
private void DetermineInputFieldValue(IInputField field, int index)
{
double result;
if (field is InputFieldCSV)
{
var fieldCSV = (InputFieldCSV)field;
ReadCSV csv = _csvMap[field];
result = csv.GetDouble(fieldCSV.Offset);
}
else if (field is InputFieldMLDataSet)
{
var mlField = (InputFieldMLDataSet)field;
MLDataFieldHolder holder = _dataSetFieldMap
[field];
IMLDataPair pair = holder.Pair;
int offset = mlField.Offset;
if (offset < pair.Input.Count)
{
result = pair.Input[offset];
}
else
{
offset -= pair.Input.Count;
result = pair.Ideal[offset];
}
}
else
{
result = field.GetValue(index);
}
field.CurrentValue = result;
return;
}
public void EvaluateNetwork()
{
BasicNetwork network = LoadNetwork();
DataNormalization norm = LoadNormalization();
var csv = new ReadCSV(_config.EvaluateFile.ToString(), false, ',');
var input = new double[norm.InputFields.Count];
var eqField = (OutputEquilateral)norm.FindOutputField(
typeof(OutputEquilateral), 0);
int correct = 0;
int total = 0;
while (csv.Next())
{
total++;
for (int i = 0; i < input.Length; i++)
{
input[i] = csv.GetDouble(i);
}
IMLData inputData = norm.BuildForNetworkInput(input);
IMLData output = network.Compute(inputData);
int coverTypeActual = DetermineTreeType(eqField, output);
int coverTypeIdeal = (int)csv.GetDouble(54) - 1;
KeepScore(coverTypeActual, coverTypeIdeal);
if (coverTypeActual == coverTypeIdeal)
{
correct++;
}
}
Console.WriteLine(@"Total cases:" + total);
Console.WriteLine(@"Correct cases:" + correct);
double percent = correct / (double)total;
Console.WriteLine(@"Correct percent:"
+ Format.FormatPercentWhole(percent));
for (int i = 0; i < 7; i++)
{
double p = (_treeCorrect[i] / (double)_treeCount[i]);
Console.WriteLine(@"Tree Type #" + i + @" - Correct/total: "
+ _treeCorrect[i] + @"/" + _treeCount[i] + @"("
+ Format.FormatPercentWhole(p) + @")");
}
}
/// <summary>
/// Process the individual training file.
/// </summary>
/// <param name="file">The training file to process.</param>
/// <param name="output">The data set to output to.</param>
protected void ProcessFile(string file, BufferedMLDataSet output)
{
var inputData = new BasicMLData(output.InputSize);
var idealData = new BasicMLData(output.IdealSize);
var csv = new ReadCSV(file, true, CSVFormat.English);
while (csv.Next())
{
var a = new double[Config.InputWindow + 1];
double close = csv.GetDouble(1);
const int fastIndex = 2;
const int slowIndex = fastIndex + Config.InputWindow;
a[0] = close;
for (int i = 0; i < 3; i++)
{
double fast = csv.GetDouble(fastIndex + i);
double slow = csv.GetDouble(slowIndex + i);
double diff = _fieldDifference.Normalize((fast - slow) / Config.PipSize);
a[i + 1] = diff;
}
_window.Add(a);
if (_window.IsFull())
{
double max = (_window.CalculateMax(0, Config.InputWindow) - close) / Config.PipSize;
double min = (_window.CalculateMin(0, Config.InputWindow) - close) / Config.PipSize;
double o = Math.Abs(max) > Math.Abs(min) ? max : min;
a = _window.GetLast();
for (int i = 0; i < 3; i++)
{
inputData[i] = a[i + 1];
}
o = _fieldOutcome.Normalize(o);
idealData[0] = o;
output.Add(inputData, idealData);
}
}
}
/// <summary>
/// Load financial data from Google.
/// </summary>
/// <param name="ticker">The ticker to load from.</param>
/// <param name="dataNeeded">The data needed.</param>
/// <param name="from">The starting time.</param>
/// <param name="to">The ending time.</param>
/// <returns>The loaded data.</returns>
public ICollection <LoadedMarketData> Load(TickerSymbol ticker, IList <MarketDataType> dataNeeded, DateTime from,
DateTime to)
{
ICollection <LoadedMarketData> result = new List <LoadedMarketData>();
Uri url = BuildUrl(ticker, from, to);
WebRequest http = WebRequest.Create(url);
var response = (HttpWebResponse)http.GetResponse();
if (response != null)
{
using (Stream istream = response.GetResponseStream())
{
var csv = new ReadCSV(istream, true, CSVFormat.DecimalPoint);
while (csv.Next())
{
DateTime date = csv.GetDate("date");
double open = csv.GetDouble("open");
double close = csv.GetDouble("close");
double high = csv.GetDouble("high");
double low = csv.GetDouble("low");
double volume = csv.GetDouble("volume");
var data =
new LoadedMarketData(date, ticker);
data.SetData(MarketDataType.Open, open);
data.SetData(MarketDataType.Close, close);
data.SetData(MarketDataType.High, high);
data.SetData(MarketDataType.Low, low);
data.SetData(MarketDataType.Open, open);
data.SetData(MarketDataType.Volume, volume);
result.Add(data);
}
csv.Close();
if (istream != null)
{
istream.Close();
}
}
}
return(result);
}
public ICollection <LoadedMarketData> ReadAndCallLoader(TickerSymbol symbol, IList <MarketDataType> neededTypes, DateTime from, DateTime to, string File)
{
try
{
//We got a file, lets load it.
ICollection <LoadedMarketData> result = new List <LoadedMarketData>();
ReadCSV csv = new ReadCSV(File, true, LoadedFormat);
csv.DateFormat = DateTimeFormat.Normalize();
// Time,Open,High,Low,Close,Volume
while (csv.Next())
{
DateTime date = csv.GetDate("Time");
double open = csv.GetDouble("Open");
double close = csv.GetDouble("High");
double high = csv.GetDouble("Low");
double low = csv.GetDouble("Close");
double volume = csv.GetDouble("Volume");
LoadedMarketData data = new LoadedMarketData(date, symbol);
data.SetData(MarketDataType.Open, open);
data.SetData(MarketDataType.High, high);
data.SetData(MarketDataType.Low, low);
data.SetData(MarketDataType.Close, close);
data.SetData(MarketDataType.Volume, volume);
result.Add(data);
}
csv.Close();
return(result);
}
catch (Exception ex)
{
Console.WriteLine("Something went wrong reading the csv");
Console.WriteLine("Something went wrong reading the csv:" + ex.Message);
}
Console.WriteLine("Something went wrong reading the csv");
return(null);
}
public void LoadCSVFileTrainingRegg()
{
var path = parameters.TrainFile;
var csvRead = new ReadCSV(new FileStream(path, FileMode.Open), true, CSVFormat.DecimalPoint);
var valuesX = new List <double[]>();
var valuesY = new List <double[]>();
while (csvRead.Next())
{
valuesX.Add(new double[1] {
csvRead.GetDouble(0)
});
valuesY.Add(new double[1] {
csvRead.GetDouble(1)
});
}
csvRead.Close();
var min = parameters.FunctionType == FunctionTypeEnum.Unipolar ? 0d : -1d;
ArgNormalize = new NormalizedField[] { new NormalizedField(NormalizationAction.Normalize, "X", valuesX.Max(v => v[0]), valuesX.Min(v => v[0]), 1.0, min) };
YNormalize = new NormalizedField(NormalizationAction.Normalize, "Y", valuesY.Max(v => v[0]), valuesY.Min(v => v[0]), 1.0, min);
for (int i = 0; i < valuesX.Count; i++)
{
valuesX[i][0] = ArgNormalize[0].Normalize(valuesX[i][0]);
valuesY[i][0] = YNormalize.Normalize(valuesY[i][0]);
}
//if(parameters.FunctionType==FunctionTypeEnum.Bipolar)
// ansMeans = means[0];
//ansfactor = factor[0];
//ansMIn = Min[0];
var trainSetCount = (int)((double)valuesX.Count * ((100.0 - 15) / 100));
valuesX.Shuffle();
valuesY.Shuffle();
MyExtensions.ResetStableShuffle();
TrainSet = new BasicNeuralDataSet(valuesX.Take(trainSetCount).ToArray(),
valuesY.Take(trainSetCount).ToArray());
ValidationSet = new BasicNeuralDataSet(valuesX.Skip(trainSetCount).ToArray(),
valuesY.Skip(trainSetCount).ToArray());
}
private static double[] ReadLine(ReadCSV csv)
{
var line = new double[csv.GetCount()];
for (int i = 0; i < csv.GetCount(); i++)
{
line[i] = csv.GetDouble(i);
}
return(line);
}
/// <summary>
/// parses one column of a csv and returns an array of doubles.
/// you can only return one double array with this method.
/// </summary>
/// <param name="file">The file.</param>
/// <param name="formatused">The formatused.</param>
/// <param name="Name">The name of the column to parse..</param>
/// <returns></returns>
public static List <double> QuickParseCSV(string file, CSVFormat formatused, string Name)
{
List <double> returnedArrays = new List <double>();
ReadCSV csv = new ReadCSV(file, true, formatused);
while (csv.Next())
{
returnedArrays.Add(csv.GetDouble(Name));
}
return(returnedArrays);
}
/// <summary>
/// Load financial data from a CSV file.
/// </summary>
/// <param name="ticker">The ticker being loaded, ignored for a CSV load.</param>
/// <param name="dataNeeded">The data needed.</param>
/// <param name="from">The starting date.</param>
/// <param name="to">The ending date.</param>
/// <returns></returns>
public ICollection <LoadedMarketData> Load(TickerSymbol ticker, IList <MarketDataType> dataNeeded, DateTime from,
DateTime to)
{
try
{
if (File.Exists(TheFile))
{
//We got a file, lets load it.
TheFile = TheFile;
ICollection <LoadedMarketData> result = new List <LoadedMarketData>();
var csv = new ReadCSV(TheFile, true, CSVFormat.English);
// Time,Open,High,Low,Close,Volume
while (csv.Next())
{
DateTime date = csv.GetDate("Time");
double open = csv.GetDouble("Open");
double close = csv.GetDouble("High");
double high = csv.GetDouble("Low");
double low = csv.GetDouble("Close");
double volume = csv.GetDouble("Volume");
var data = new LoadedMarketData(date, ticker);
data.SetData(MarketDataType.Open, open);
data.SetData(MarketDataType.Volume, close);
data.SetData(MarketDataType.High, high);
data.SetData(MarketDataType.Low, low);
data.SetData(MarketDataType.Volume, volume);
result.Add(data);
}
csv.Close();
return(result);
}
}
catch (Exception ex)
{
throw new LoaderError(ex);
}
throw new LoaderError(@"Something went wrong reading the csv");
}
/// <summary>
/// Read one record of data from a CSV file.
/// </summary>
/// <param name="input">The input data array.</param>
/// <param name="ideal">The ideal data array.</param>
/// <returns>True, if there is more data to be read.</returns>
public bool Read(double[] input, double[] ideal)
{
if (this.readCSV.Next())
{
int index = 0;
for (int i = 0; i < input.Length; i++)
{
input[i] = readCSV.GetDouble(index++);
}
for (int i = 0; i < ideal.Length; i++)
{
ideal[i] = readCSV.GetDouble(index++);
}
return(true);
}
else
{
return(false);
}
}
/// <summary>
/// parses one column of a csv and returns an array of doubles.
/// you can only return one double array with this method.
/// We are assuming CSVFormat english in this quick parse csv method.
/// You can input the size (number of lines) to read.
/// </summary>
/// <param name="file">The file.</param>
/// <param name="Name">The name of the column to parse.</param>
/// <param name="size">The size.</param>
/// <returns></returns>
public static List <double> QuickParseCSV(string file, string Name, int size)
{
List <double> returnedArrays = new List <double>();
ReadCSV csv = new ReadCSV(file, true, CSVFormat.English);
int currentRead = 0;
while (csv.Next() && currentRead < size)
{
returnedArrays.Add(csv.GetDouble(Name));
currentRead++;
}
return(returnedArrays);
}
/// <summary>
/// Used to calibrate the training file.
/// </summary>
/// <param name="file">The file to consider.</param>
protected void CalibrateFile(string file)
{
var csv = new ReadCSV(file, true, CSVFormat.English);
while (csv.Next())
{
var a = new double[1];
double close = csv.GetDouble(1);
const int fastIndex = 2;
const int slowIndex = fastIndex + Config.InputWindow;
a[0] = close;
for (int i = 0; i < Config.InputWindow; i++)
{
double fast = csv.GetDouble(fastIndex + i);
double slow = csv.GetDouble(slowIndex + i);
if (!double.IsNaN(fast) && !double.IsNaN(slow))
{
double diff = (fast - slow) / Config.PipSize;
_minDifference = Math.Min(_minDifference, diff);
_maxDifference = Math.Max(_maxDifference, diff);
}
}
_window.Add(a);
if (_window.IsFull())
{
double max = (_window.CalculateMax(0, Config.InputWindow) - close) / Config.PipSize;
double min = (_window.CalculateMin(0, Config.InputWindow) - close) / Config.PipSize;
double o = Math.Abs(max) > Math.Abs(min) ? max : min;
_maxPiPs = Math.Max(_maxPiPs, (int)o);
_minPiPs = Math.Min(_minPiPs, (int)o);
}
}
}
public void loadPrime(String primeFilename)
{
ReadCSV csv = new ReadCSV(primeFilename);
while (csv.Next())
{
DateTime date = csv.GetDate("date");
double rate = csv.GetDouble("prime");
InterestRate ir = new InterestRate(date, rate);
this.rates.Add(ir);
}
csv.Close();
this.rates.Sort();
}
public void loadPrime(String primeFilename)
{
var csv = new ReadCSV(primeFilename, true, CSVFormat.English);
while (csv.Next())
{
var date = csv.GetDate("date");
double rate = csv.GetDouble("prime");
var ir = new InterestRate(date, rate);
rates.Add(ir);
}
csv.Close();
rates.Sort();
}
public void loadSP500(String sp500Filename)
{
ReadCSV csv = new ReadCSV(sp500Filename);
while (csv.Next())
{
DateTime date = csv.GetDate("date");
double amount = csv.GetDouble("adj close");
FinancialSample sample = new FinancialSample();
sample.setAmount(amount);
sample.setDate(date);
this.samples.Add(sample);
}
csv.Close();
this.samples.Sort();
}
public void loadSP500(String sp500Filename)
{
var csv = new ReadCSV(sp500Filename, true, CSVFormat.English);
while (csv.Next())
{
var date = csv.GetDate("date");
double amount = csv.GetDouble("adj close");
var sample = new FinancialSample();
sample.setAmount(amount);
sample.setDate(date);
samples.Add(sample);
}
csv.Close();
samples.Sort();
}
/// <summary>
/// Load financial data.
/// </summary>
/// <param name="ticker">The ticker symbol.</param>
/// <param name="output">The output file.</param>
/// <param name="outputFormat">The output format.</param>
/// <param name="from">Starting date.</param>
/// <param name="to">Ending date.</param>
public void LoadAllData(String ticker, String output, CSVFormat outputFormat, DateTime from,
DateTime to)
{
try
{
Uri urlData = BuildURL(ticker, from, to);
WebRequest httpData = WebRequest.Create(urlData);
var responseData = (HttpWebResponse)httpData.GetResponse();
if (responseData != null)
{
Stream istreamData = responseData.GetResponseStream();
var csvData = new ReadCSV(istreamData, true, CSVFormat.English);
TextWriter tw = new StreamWriter(output);
tw.WriteLine("date,time,open price,high price,low price,close price,volume,adjusted price");
while (csvData.Next())
{
DateTime date = csvData.GetDate("date");
double adjustedClose = csvData.GetDouble("adj close");
double open = csvData.GetDouble("open");
double close = csvData.GetDouble("close");
double high = csvData.GetDouble("high");
double low = csvData.GetDouble("low");
var volume = (long)csvData.GetDouble("volume");
var line = new StringBuilder();
line.Append(NumericDateUtil.DateTime2Long(date));
line.Append(outputFormat.Separator);
line.Append(NumericDateUtil.Time2Int(date));
line.Append(outputFormat.Separator);
line.Append(outputFormat.Format(open, Precision));
line.Append(outputFormat.Separator);
line.Append(outputFormat.Format(high, Precision));
line.Append(outputFormat.Separator);
line.Append(outputFormat.Format(low, Precision));
line.Append(outputFormat.Separator);
line.Append(outputFormat.Format(close, Precision));
line.Append(outputFormat.Separator);
line.Append(volume);
line.Append(outputFormat.Separator);
line.Append(outputFormat.Format(adjustedClose, Precision));
tw.WriteLine(line.ToString());
}
tw.Close();
}
}
catch (WebException ex)
{
throw new QuantError(ex);
}
}
/// <summary>
/// parses one column of a csv and returns an array of doubles.
/// you can only return one double array with this method.
/// We are assuming CSVFormat english in this quick parse csv method.
/// You can input the size (number of lines) to read , and the number of lines you want to skip start from the first line.
/// </summary>
/// <param name="file">The file.</param>
/// <param name="columnNumber">The column number to get.</param>
/// <param name="size">The size.</param>
/// <param name="startLine">The start line (how many lines you want to skip from the first line.</param>
/// <returns></returns>
public static List <double> QuickParseCSV(string file, int columnNumber, int size, int startLine)
{
List <double> returnedArrays = new List <double>();
ReadCSV csv = new ReadCSV(file, true, CSVFormat.English);
int currentRead = 0;
int currentLine = 0;
while (csv.Next())
{
if (currentRead < size && currentLine > startLine)
{
returnedArrays.Add(csv.GetDouble(columnNumber));
currentRead++;
}
currentLine++;
}
return(returnedArrays);
}
/// <summary>
/// Loads the specified financial data.
/// </summary>
/// <param name="ticker">The currency pair to load.</param>
/// <param name="dataNeeded">The financial data needed.</param>
/// <param name="from">The beginning date to load data from.</param>
/// <param name="to">The ending date to load data to.</param>
/// <returns>A collection of LoadedMarketData objects that represent
/// the data loaded.</returns>
public ICollection <LoadedMarketData> Load(
TickerSymbol ticker,
IList <MarketDataType> dataNeeded,
DateTime from,
DateTime to
)
{
ICollection <LoadedMarketData> result =
new List <LoadedMarketData>();
Uri url = buildURL(ticker, from, to);
WebRequest http = HttpWebRequest.Create(url);
HttpWebResponse response = http.GetResponse() as HttpWebResponse;
using (Stream istream = response.GetResponseStream())
{
ReadCSV csv =
new ReadCSV(istream, true, CSVFormat.DECIMAL_POINT);
while (csv.Next())
{
// TODO: check these values if possible
DateTime date = csv.GetDate("date");
double adjClose = csv.GetDouble("adj close"); // TODO: deprecate?
double open = csv.GetDouble("open");
double close = csv.GetDouble("close");
double high = csv.GetDouble("high");
double low = csv.GetDouble("low");
double volume = csv.GetDouble("volume"); // TODO: deprecate?
LoadedMarketData data =
new LoadedMarketData(date, ticker);
data.SetData(MarketDataType.ADJUSTED_CLOSE, adjClose);
data.SetData(MarketDataType.OPEN, open);
data.SetData(MarketDataType.CLOSE, close);
data.SetData(MarketDataType.HIGH, high);
data.SetData(MarketDataType.LOW, low);
data.SetData(MarketDataType.OPEN, open);
data.SetData(MarketDataType.VOLUME, volume);
result.Add(data);
}
csv.Close();
istream.Close();
}
return(result);
}
/// <summary>
/// Called internally to obtain the current value for an input field.
/// </summary>
/// <param name="field">The input field to determine.</param>
/// <param name="index">The current index.</param>
/// <returns>The value for this input field.</returns>
private double DetermineInputFieldValue(IInputField field,
int index)
{
double result = 0;
if (field is InputFieldCSV)
{
InputFieldCSV fieldCSV = (InputFieldCSV)field;
ReadCSV csv = this.csvMap[field];
result = csv.GetDouble(fieldCSV.Offset);
}
else if (field is InputFieldNeuralDataSet)
{
InputFieldNeuralDataSet neuralField = (InputFieldNeuralDataSet)field;
NeuralDataFieldHolder holder = this.dataSetFieldMap
[field];
INeuralDataPair pair = holder.Pair;
int offset = neuralField.Offset;
if (offset < pair.Input.Count)
{
result = pair.Input[offset];
}
else
{
offset -= pair.Input.Count;
result = pair.Ideal[offset];
}
}
else
{
result = field.GetValue(index);
}
field.CurrentValue = result;
return(result);
}
/// <summary>
/// Load the specified financial data.
/// </summary>
/// <param name="ticker">The ticker symbol to load.</param>
/// <param name="dataNeeded">The financial data needed.</param>
/// <param name="from">The beginning date to load data from.</param>
/// <param name="to">The ending date to load data to.</param>
/// <returns>A collection of LoadedMarketData objects that represent the data
/// loaded.</returns>
public ICollection<LoadedMarketData> Load(TickerSymbol ticker,
IList<MarketDataType> dataNeeded, DateTime from,
DateTime to)
{
ICollection<LoadedMarketData> result =
new List<LoadedMarketData>();
Uri url = BuildURL(ticker, from, to);
WebRequest http = WebRequest.Create(url);
var response = (HttpWebResponse) http.GetResponse();
using (Stream istream = response.GetResponseStream())
{
var csv = new ReadCSV(istream, true, CSVFormat.DecimalPoint);
while (csv.Next())
{
DateTime date = csv.GetDate("date");
double adjClose = csv.GetDouble("adj close");
double open = csv.GetDouble("open");
double close = csv.GetDouble("close");
double high = csv.GetDouble("high");
double low = csv.GetDouble("low");
double volume = csv.GetDouble("volume");
var data =
new LoadedMarketData(date, ticker);
data.SetData(MarketDataType.AdjustedClose, adjClose);
data.SetData(MarketDataType.Open, open);
data.SetData(MarketDataType.Close, close);
data.SetData(MarketDataType.High, high);
data.SetData(MarketDataType.Low, low);
data.SetData(MarketDataType.Open, open);
data.SetData(MarketDataType.Volume, volume);
result.Add(data);
}
csv.Close();
istream.Close();
}
return result;
}
/// <summary>
/// Load financial data.
/// </summary>
/// <param name="ticker">The ticker symbol.</param>
/// <param name="output">The output file.</param>
/// <param name="outputFormat">The output format.</param>
/// <param name="from">Starting date.</param>
/// <param name="to">Ending date.</param>
public void LoadAllData(String ticker, String output, CSVFormat outputFormat, DateTime from,
DateTime to)
{
try
{
Uri urlData = BuildURL(ticker, from, to);
WebRequest httpData = WebRequest.Create(urlData);
var responseData = (HttpWebResponse) httpData.GetResponse();
if (responseData != null)
{
Stream istreamData = responseData.GetResponseStream();
var csvData = new ReadCSV(istreamData, true, CSVFormat.English);
TextWriter tw = new StreamWriter(output);
tw.WriteLine("date,time,open price,high price,low price,close price,volume,adjusted price");
while (csvData.Next())
{
DateTime date = csvData.GetDate("date");
double adjustedClose = csvData.GetDouble("adj close");
double open = csvData.GetDouble("open");
double close = csvData.GetDouble("close");
double high = csvData.GetDouble("high");
double low = csvData.GetDouble("low");
var volume = (long) csvData.GetDouble("volume");
var line = new StringBuilder();
line.Append(NumericDateUtil.DateTime2Long(date));
line.Append(outputFormat.Separator);
line.Append(NumericDateUtil.Time2Int(date));
line.Append(outputFormat.Separator);
line.Append(outputFormat.Format(open, Precision));
line.Append(outputFormat.Separator);
line.Append(outputFormat.Format(high, Precision));
line.Append(outputFormat.Separator);
line.Append(outputFormat.Format(low, Precision));
line.Append(outputFormat.Separator);
line.Append(outputFormat.Format(close, Precision));
line.Append(outputFormat.Separator);
line.Append(volume);
line.Append(outputFormat.Separator);
line.Append(outputFormat.Format(adjustedClose, Precision));
tw.WriteLine(line.ToString());
}
tw.Close();
}
}
catch (WebException ex)
{
throw new QuantError(ex);
}
}
public ICollection<LoadedMarketData> Load(
TickerSymbol ticker,
IList<MarketDataType> dataNeeded,
DateTime from,
DateTime to)
{
// TODO: nyyyyyyyaaagh!
ICollection<LoadedMarketData> result =
new List<LoadedMarketData>();
Uri url = BuildURL(ticker, from, to);
WebRequest http = HttpWebRequest.Create(url);
HttpWebResponse response = http.GetResponse() as HttpWebResponse;
using (Stream istream = response.GetResponseStream())
{
ReadCSV csv = new ReadCSV(
istream,
true,
CSVFormat.DECIMAL_POINT
);
while (csv.Next())
{
// todo: edit headers to match
DateTime date = csv.GetDate("DATE");
date =
date.Add(
new TimeSpan(
csv.GetDate("TIME").Hour,
csv.GetDate("TIME").Minute,
csv.GetDate("TIME").Second
)
);
double open = csv.GetDouble("OPEN");
double high = csv.GetDouble("MIN");
double low = csv.GetDouble("MAX");
double close = csv.GetDouble("CLOSE");
double volume = csv.GetDouble("VOLUME");
LoadedMarketData data =
new LoadedMarketData(date, ticker);
data.SetData(MarketDataType.OPEN, open);
data.SetData(MarketDataType.HIGH, high);
data.SetData(MarketDataType.LOW, low);
data.SetData(MarketDataType.CLOSE, close);
data.SetData(MarketDataType.VOLUME, volume);
result.Add(data);
}
csv.Close();
istream.Close();
}
return result;
}
/// <summary>
/// Load the specified financial data.
/// </summary>
/// <param name="ticker">The ticker symbol to load.</param>
/// <param name="dataNeeded">The financial data needed.</param>
/// <param name="from">The beginning date to load data from.</param>
/// <param name="to">The ending date to load data to.</param>
/// <returns>A collection of LoadedMarketData objects that represent the data
/// loaded.</returns>
public ICollection<LoadedMarketData> Load(TickerSymbol ticker,
IList<MarketDataType> dataNeeded, DateTime from,
DateTime to)
{
ICollection<LoadedMarketData> result =
new List<LoadedMarketData>();
Uri url = buildURL(ticker, from, to);
WebRequest http = HttpWebRequest.Create(url);
HttpWebResponse response = (HttpWebResponse)http.GetResponse();
using (Stream istream = response.GetResponseStream())
{
ReadCSV csv = new ReadCSV(istream, true, CSVFormat.DECIMAL_POINT);
while (csv.Next())
{
DateTime date = csv.GetDate("date");
double adjClose = csv.GetDouble("adj close");
double open = csv.GetDouble("open");
double close = csv.GetDouble("close");
double high = csv.GetDouble("high");
double low = csv.GetDouble("low");
double volume = csv.GetDouble("volume");
LoadedMarketData data =
new LoadedMarketData(date, ticker);
data.SetData(MarketDataType.ADJUSTED_CLOSE, adjClose);
data.SetData(MarketDataType.OPEN, open);
data.SetData(MarketDataType.CLOSE, close);
data.SetData(MarketDataType.HIGH, high);
data.SetData(MarketDataType.LOW, low);
data.SetData(MarketDataType.OPEN, open);
data.SetData(MarketDataType.VOLUME, volume);
result.Add(data);
}
csv.Close();
istream.Close();
}
return result;
}
public void EvaluateNetwork()
{
BasicNetwork network = LoadNetwork();
DataNormalization norm = LoadNormalization();
var csv = new ReadCSV(_config.EvaluateFile.ToString(), false, ',');
var input = new double[norm.InputFields.Count];
var eqField = (OutputEquilateral) norm.FindOutputField(
typeof (OutputEquilateral), 0);
int correct = 0;
int total = 0;
while (csv.Next())
{
total++;
for (int i = 0; i < input.Length; i++)
{
input[i] = csv.GetDouble(i);
}
IMLData inputData = norm.BuildForNetworkInput(input);
IMLData output = network.Compute(inputData);
int coverTypeActual = DetermineTreeType(eqField, output);
int coverTypeIdeal = (int) csv.GetDouble(54) - 1;
KeepScore(coverTypeActual, coverTypeIdeal);
if (coverTypeActual == coverTypeIdeal)
{
correct++;
}
}
Console.WriteLine(@"Total cases:" + total);
Console.WriteLine(@"Correct cases:" + correct);
double percent = correct/(double) total;
Console.WriteLine(@"Correct percent:"
+ Format.FormatPercentWhole(percent));
for (int i = 0; i < 7; i++)
{
double p = (_treeCorrect[i]/(double) _treeCount[i]);
Console.WriteLine(@"Tree Type #" + i + @" - Correct/total: "
+ _treeCorrect[i] + @"/" + _treeCount[i] + @"("
+ Format.FormatPercentWhole(p) + @")");
}
}
/// <summary>
/// Reads and parses CSV data from file
/// </summary>
/// <param name="ticker">Ticker associated with CSV file</param>
/// <param name="neededTypes">Columns to parse (headers)</param>
/// <param name="from">DateTime from</param>
/// <param name="to">DateTime to</param>
/// <param name="File">Filepath to CSV</param>
/// <returns>Marketdata</returns>
public ICollection <LoadedMarketData> ReadAndCallLoader(TickerSymbol ticker, IList <MarketDataType> neededTypes, DateTime from, DateTime to, string File)
{
try
{
LoadedFile = File;
Console.WriteLine("Loading instrument: " + ticker.Symbol + " from: " + File);
//We got a file, lets load it.
ICollection <LoadedMarketData> result = new List <LoadedMarketData>();
ReadCSV csv = new ReadCSV(File, true, LoadedFormat);
if (DateTimeDualColumn)
{
csv.DateFormat = DateFormat;
csv.TimeFormat = TimeFormat;
}
else
{
csv.DateFormat = DateFormat;
}
//"Date","Time","Open","High","Low","Close","Volume"
while (csv.Next())
{
string datetime = "";
if (DateTimeDualColumn)
{
datetime = csv.GetDate("Date").ToShortDateString() + " " +
csv.GetTime("Time").ToShortTimeString();
}
else
{
datetime = csv.GetDate("Date").ToShortDateString();
}
DateTime date = DateTime.Parse(datetime);
if (date > from && date < to)
{
// CSV columns
double open = csv.GetDouble("Open");
double high = csv.GetDouble("High");
double low = csv.GetDouble("Low");
double close = csv.GetDouble("Close");
double volume = csv.GetDouble("Volume");
LoadedMarketData data = new LoadedMarketData(date, ticker);
foreach (MarketDataType marketDataType in neededTypes)
{
switch (marketDataType.ToString())
{
case "Open":
data.SetData(MarketDataType.Open, open);
break;
case "High":
data.SetData(MarketDataType.High, high);
break;
case "Low":
data.SetData(MarketDataType.Low, low);
break;
case "Close":
data.SetData(MarketDataType.Close, close);
break;
case "Volume":
data.SetData(MarketDataType.Volume, volume);
break;
case "RangeHighLow":
data.SetData(MarketDataType.RangeHighLow, Math.Round(Math.Abs(high - low), 6));
break;
case "RangeOpenClose":
data.SetData(MarketDataType.RangeOpenClose, Math.Round(Math.Abs(close - open), 6));
break;
case "RangeOpenCloseNonAbsolute":
data.SetData(MarketDataType.RangeOpenCloseNonAbsolute, Math.Round(close - open, 6));
break;
case "Weighted":
data.SetData(MarketDataType.Weighted, Math.Round((high + low + 2 * close) / 4, 6));
break;
}
}
result.Add(data);
}
}
csv.Close();
return(result);
}
catch (Exception ex)
{
Console.WriteLine("Something went wrong reading the csv: " + ex.Message);
}
return(null);
}
public ICollection<LoadedMarketData> Load(TickerSymbol ticker, IList<MarketDataType> dataNeeded, DateTime from, DateTime to)
{
ICollection<LoadedMarketData> is2 = new List<LoadedMarketData>();
HttpWebResponse response = (HttpWebResponse) WebRequest.Create(x38c212309d8d5dd3(ticker, from, to)).GetResponse();
using (Stream stream = response.GetResponseStream())
{
DateTime time;
double num;
double num2;
double num3;
double num4;
double num5;
double num6;
LoadedMarketData data;
ReadCSV dcsv = new ReadCSV(stream, true, CSVFormat.DecimalPoint);
goto Label_005B;
Label_003F:
data.SetData(MarketDataType.Open, num2);
Label_0049:
data.SetData(MarketDataType.Volume, num6);
is2.Add(data);
Label_005B:
if (dcsv.Next())
{
goto Label_01AF;
}
dcsv.Close();
stream.Close();
if ((((uint) num) - ((uint) num5)) <= uint.MaxValue)
{
goto Label_0125;
}
if ((((uint) num) & 0) == 0)
{
goto Label_00B0;
}
Label_00A4:
data.SetData(MarketDataType.Low, num5);
goto Label_003F;
Label_00B0:
data.SetData(MarketDataType.AdjustedClose, num);
do
{
data.SetData(MarketDataType.Open, num2);
if ((((uint) num) - ((uint) num6)) > uint.MaxValue)
{
goto Label_0049;
}
data.SetData(MarketDataType.Close, num3);
data.SetData(MarketDataType.High, num4);
}
while ((((uint) num5) - ((uint) num3)) > uint.MaxValue);
if (((uint) num2) >= 0)
{
goto Label_00A4;
}
goto Label_003F;
Label_0125:
if (((uint) num2) >= 0)
{
return is2;
}
goto Label_005B;
Label_013C:
num6 = dcsv.GetDouble("volume");
data = new LoadedMarketData(time, ticker);
if ((((uint) num2) | 2) == 0)
{
goto Label_017C;
}
goto Label_00B0;
Label_016E:
num2 = dcsv.GetDouble("open");
Label_017C:
num3 = dcsv.GetDouble("close");
num4 = dcsv.GetDouble("high");
num5 = dcsv.GetDouble("low");
goto Label_013C;
Label_01AF:
time = dcsv.GetDate("date");
num = dcsv.GetDouble("adj close");
if ((((uint) num3) + ((uint) num6)) <= uint.MaxValue)
{
goto Label_016E;
}
goto Label_00B0;
}
}
