public void TestCSVLoader()
{
var loader = new CSVFinal();
loader.DateFormat = "yyyy.MM.dd hh:mm:ss";
var tickerAAPL = new TickerSymbol("AAPL", "NY");
var desc = new MarketDataDescription(tickerAAPL, MarketDataType.Close, true, true);
MarketMLDataSet marketData = new MarketMLDataSet(loader, 5, 1);
marketData.AddDescription(desc);
marketData.SequenceGrandularity = Util.Time.TimeUnit.Hours;
var begin = new DateTime(2006, 1, 1);
var end = new DateTime(2007, 7, 31);
loader.GetFile((AssemblyDirectory + "\\smallCSV.csv"));
marketData.Load(begin, end);
marketData.Generate();
// first test the points
IEnumerator<TemporalPoint> itr = marketData.Points.GetEnumerator();
itr.MoveNext();
TemporalPoint point = itr.Current;
Assert.AreEqual(0, point.Sequence);
Assert.AreEqual(1, point.Data.Length);
Assert.AreEqual(1.12884, point[0]);
Assert.AreEqual(5, marketData.Points.Count);
}
public void TestCSVLoader()
{
var loader = new CSVFinal();
loader.DateFormat = "yyyy.MM.dd hh:mm:ss";
var tickerAAPL = new TickerSymbol("AAPL", "NY");
var desc = new MarketDataDescription(tickerAAPL, MarketDataType.Close, true, true);
MarketMLDataSet marketData = new MarketMLDataSet(loader, 5, 1);
marketData.AddDescription(desc);
marketData.SequenceGrandularity = Util.Time.TimeUnit.Hours;
var begin = new DateTime(2006, 1, 1);
var end = new DateTime(2007, 7, 31);
loader.GetFile((AssemblyDirectory + "\\smallCSV.csv"));
marketData.Load(begin, end);
marketData.Generate();
// first test the points
IEnumerator <TemporalPoint> itr = marketData.Points.GetEnumerator();
itr.MoveNext();
TemporalPoint point = itr.Current;
Assert.AreEqual(0, point.Sequence);
Assert.AreEqual(1, point.Data.Length);
Assert.AreEqual(1.12884, point[0]);
Assert.AreEqual(5, marketData.Points.Count);
}
public static void Generate(FileInfo dataDir)
{
IMarketLoader loader = new YahooFinanceLoader();
var market = new MarketMLDataSet(loader,
Config.INPUT_WINDOW, Config.PREDICT_WINDOW);
var desc = new MarketDataDescription(
Config.TICKER, MarketDataType.AdjustedClose, true, true);
market.AddDescription(desc);
var end = DateTime.Now; // end today
var begin = new DateTime(end.Ticks); // begin 30 days ago
// Gather training data for the last 2 years, stopping 60 days short of today.
// The 60 days will be used to evaluate prediction.
begin = begin.AddDays(-60);
end = end.AddDays(-60);
begin = begin.AddYears(-2);
market.Load(begin, end);
market.Generate();
EncogUtility.SaveEGB(FileUtil.CombinePath(dataDir, Config.TRAINING_FILE), market);
// create a network
BasicNetwork network = EncogUtility.SimpleFeedForward(
market.InputSize,
Config.HIDDEN1_COUNT,
Config.HIDDEN2_COUNT,
market.IdealSize,
true);
// save the network and the training
EncogDirectoryPersistence.SaveObject(FileUtil.CombinePath(dataDir, Config.NETWORK_FILE), network);
}
public static MarketMLDataSet GrabData()
{
IMarketLoader loader = new YahooFinanceLoader();
var result = new MarketMLDataSet(loader,
Config.INPUT_WINDOW, Config.PREDICT_WINDOW);
var desc = new MarketDataDescription(Config.TICKER,
MarketDataType.AdjustedClose, true, true);
result.AddDescription(desc);
var end = DateTime.Now; // end today
var begin = new DateTime(end.Ticks); // begin 30 days ago
begin = begin.AddDays(-60);
result.Load(begin, end);
result.Generate();
return result;
}
public static void Generate(string fileName)
{
FileInfo dataDir = new FileInfo(@Environment.CurrentDirectory);
IMarketLoader loader = new CSVFinal();
var market = new MarketMLDataSet(loader,CONFIG.INPUT_WINDOW, CONFIG.PREDICT_WINDOW);
// var desc = new MarketDataDescription(Config.TICKER, MarketDataType.Close, true, true);
var desc = new MarketDataDescription(CONFIG.TICKER, MarketDataType.Close, true, true);
market.AddDescription(desc);
string currentDirectory =@"c:\";
loader.GetFile(fileName);
var end = DateTime.Now; // end today
var begin = new DateTime(end.Ticks); // begin 30 days ago
// Gather training data for the last 2 years, stopping 60 days short of today.
// The 60 days will be used to evaluate prediction.
begin = begin.AddDays(-600);
end = begin.AddDays(200);
Console.WriteLine("You are loading date from:" + begin.ToShortDateString() + " To :" + end.ToShortDateString());
market.Load(begin, end);
market.Generate();
EncogUtility.SaveEGB(FileUtil.CombinePath(dataDir, CONFIG.SVMTRAINING_FILE), market);
// create a network
//BasicNetwork network = EncogUtility.SimpleFeedForward(
// market.InputSize,
// CONFIG.HIDDEN1_COUNT,
// CONFIG.HIDDEN2_COUNT,
// market.IdealSize,
// true);
SupportVectorMachine network = new SupportVectorMachine(CONFIG.INPUT_WINDOW, true);
TrainNetworks(network, market);
// save the network and the training
EncogDirectoryPersistence.SaveObject(FileUtil.CombinePath(dataDir,CONFIG.SVMTRAINING_FILE), network);
}
public static MarketMLDataSet GrabData(string newfileLoad)
{
IMarketLoader loader = new CSVLoader();
loader.GetFile(newfileLoad);
var result = new MarketMLDataSet(loader,
Config.INPUT_WINDOW, Config.PREDICT_WINDOW);
// var desc = new MarketDataDescription(Config.TICKER,
// MarketDataType.Close, true, true);
var desc = new MarketDataDescription(Config.TICKER,
MarketDataType.Trade, true, true);
result.AddDescription(desc);
var end = DateTime.Now; // end today
var begin = new DateTime(end.Ticks); // begin 30 days ago
begin = begin.AddDays(-150);
result.Load(begin, end);
result.Generate();
return result;
}
public void MarketData()
{
IMarketLoader loader = new YahooFinanceLoader();
var tickerAAPL = new TickerSymbol("AAPL", null);
var tickerMSFT = new TickerSymbol("MSFT", null);
var marketData = new MarketMLDataSet(loader, 5, 1);
marketData.AddDescription(new MarketDataDescription(tickerAAPL, MarketDataType.Close, true, true));
marketData.AddDescription(new MarketDataDescription(tickerMSFT, MarketDataType.Close, true, false));
marketData.AddDescription(new MarketDataDescription(tickerAAPL, MarketDataType.Volume, true, false));
marketData.AddDescription(new MarketDataDescription(tickerMSFT, MarketDataType.Volume, true, false));
var begin = new DateTime(2008, 7, 1);
var end = new DateTime(2008, 7, 31);
marketData.Load(begin, end);
marketData.Generate();
Assert.AreEqual(22, marketData.Points.Count);
// first test the points
IEnumerator<TemporalPoint> itr = marketData.Points.GetEnumerator();
itr.MoveNext();
TemporalPoint point = itr.Current;
Assert.AreEqual(0, point.Sequence);
Assert.AreEqual(4, point.Data.Length);
Assert.AreEqual(174.68, point[0]);
Assert.AreEqual(26.87, point[1]);
Assert.AreEqual(39, (int) (point[2]/1000000));
Assert.AreEqual(100, (int) (point[3]/1000000));
itr.MoveNext();
point = itr.Current;
Assert.AreEqual(1, point.Sequence);
Assert.AreEqual(4, point.Data.Length);
Assert.AreEqual(168.18, point[0]);
Assert.AreEqual(25.88, point[1]);
Assert.AreEqual(29, (int) (point[2]/1000000));
Assert.AreEqual(84, (int) (point[3]/1000000));
itr.MoveNext();
point = itr.Current;
Assert.AreEqual(2, point.Sequence);
Assert.AreEqual(4, point.Data.Length);
Assert.AreEqual(170.12, point[0]);
Assert.AreEqual(25.98, point[1]);
Assert.AreEqual(18, (int) (point[2]/1000000));
Assert.AreEqual(37, (int) (point[3]/1000000));
// now check the actual data
Assert.AreEqual(16, marketData.Data.Count);
Assert.AreEqual(20, marketData.InputNeuronCount);
Assert.AreEqual(1, marketData.OutputNeuronCount);
IEnumerator<IMLDataPair> itr2 = marketData.Data.GetEnumerator();
itr2.MoveNext();
IMLDataPair pair = itr2.Current;
Assert.AreEqual(20, pair.Input.Count);
Assert.AreEqual(1, pair.Ideal.Count);
Assert.AreEqual(-0.037, Math.Round(pair.Input[0]*1000.0)/1000.0);
Assert.AreEqual(-0.037, Math.Round(pair.Input[1]*1000.0)/1000.0);
Assert.AreEqual(-0.246, Math.Round(pair.Input[2]*1000.0)/1000.0);
Assert.AreEqual(-0.156, Math.Round(pair.Input[3]*1000.0)/1000.0);
Assert.AreEqual(0.012, Math.Round(pair.Input[4]*1000.0)/1000.0);
Assert.AreEqual(0.0040, Math.Round(pair.Input[5]*1000.0)/1000.0);
Assert.AreEqual(-0.375, Math.Round(pair.Input[6]*1000.0)/1000.0);
Assert.AreEqual(-0.562, Math.Round(pair.Input[7]*1000.0)/1000.0);
Assert.AreEqual(0.03, Math.Round(pair.Input[8]*1000.0)/1000.0);
Assert.AreEqual(0.0020, Math.Round(pair.Input[9]*1000.0)/1000.0);
Assert.AreEqual(0.57, Math.Round(pair.Input[10]*100.0)/100.0);
Assert.AreEqual(0.929, Math.Round(pair.Input[11]*1000.0)/1000.0);
Assert.AreEqual(0.025, Math.Round(pair.Input[12]*1000.0)/1000.0);
Assert.AreEqual(-0.0070, Math.Round(pair.Input[13]*1000.0)/1000.0);
// for some reason, Yahoo likes to vary the volume numbers slightly, sometimes!
Assert.AreEqual(0.1, Math.Round(pair.Input[14]*10.0)/10.0);
Assert.AreEqual(-0.084, Math.Round(pair.Input[15]*1000.0)/1000.0);
Assert.AreEqual(-0.03, Math.Round(pair.Input[16]*1000.0)/1000.0);
Assert.AreEqual(-0.024, Math.Round(pair.Input[17]*1000.0)/1000.0);
Assert.AreEqual(0.008, Math.Round(pair.Input[18]*1000.0)/1000.0);
Assert.AreEqual(-0.172, Math.Round(pair.Input[19]*1000.0)/1000.0);
itr2.MoveNext();
pair = itr2.Current;
Assert.AreEqual(20, pair.Input.Count);
Assert.AreEqual(1, pair.Ideal.Count);
Assert.AreEqual(0.012, Math.Round(pair.Input[0]*1000.0)/1000.0);
Assert.AreEqual(0.0040, Math.Round(pair.Input[1]*1000.0)/1000.0);
Assert.AreEqual(-0.375, Math.Round(pair.Input[2]*1000.0)/1000.0);
Assert.AreEqual(-0.562, Math.Round(pair.Input[3]*1000.0)/1000.0);
Assert.AreEqual(0.03, Math.Round(pair.Input[4]*1000.0)/1000.0);
Assert.AreEqual(0.0020, Math.Round(pair.Input[5]*1000.0)/1000.0);
Assert.AreEqual(0.6, Math.Round(pair.Input[6]*10.0)/10.0);
Assert.AreEqual(0.929, Math.Round(pair.Input[7]*1000.0)/1000.0);
Assert.AreEqual(0.025, Math.Round(pair.Input[8]*1000.0)/1000.0);
Assert.AreEqual(-0.0070, Math.Round(pair.Input[9]*1000.0)/1000.0);
Assert.AreEqual(0.1, Math.Round(pair.Input[10]*10.0)/10.0);
Assert.AreEqual(-0.084, Math.Round(pair.Input[11]*1000.0)/1000.0);
Assert.AreEqual(-0.03, Math.Round(pair.Input[12]*1000.0)/1000.0);
Assert.AreEqual(-0.024, Math.Round(pair.Input[13]*1000.0)/1000.0);
Assert.AreEqual(0.0080, Math.Round(pair.Input[14]*1000.0)/1000.0);
Assert.AreEqual(-0.172, Math.Round(pair.Input[15]*1000.0)/1000.0);
Assert.AreEqual(0.014, Math.Round(pair.Input[16]*1000.0)/1000.0);
Assert.AreEqual(0.0090, Math.Round(pair.Input[17]*1000.0)/1000.0);
Assert.AreEqual(-0.1, Math.Round(pair.Input[18]*10.0)/10.0);
Assert.AreEqual(0.066, Math.Round(pair.Input[19]*1000.0)/1000.0);
}
public void MarketData()
{
IMarketLoader loader = new YahooFinanceLoader();
var tickerAAPL = new TickerSymbol("AAPL", null);
var tickerMSFT = new TickerSymbol("MSFT", null);
var marketData = new MarketMLDataSet(loader, 5, 1);
marketData.AddDescription(new MarketDataDescription(tickerAAPL, MarketDataType.Close, true, true));
marketData.AddDescription(new MarketDataDescription(tickerMSFT, MarketDataType.Close, true, false));
marketData.AddDescription(new MarketDataDescription(tickerAAPL, MarketDataType.Volume, true, false));
marketData.AddDescription(new MarketDataDescription(tickerMSFT, MarketDataType.Volume, true, false));
var begin = new DateTime(2008, 7, 1);
var end = new DateTime(2008, 7, 31);
marketData.Load(begin, end);
marketData.Generate();
Assert.AreEqual(22, marketData.Points.Count);
// first test the points
IEnumerator <TemporalPoint> itr = marketData.Points.GetEnumerator();
itr.MoveNext();
TemporalPoint point = itr.Current;
Assert.AreEqual(0, point.Sequence);
Assert.AreEqual(4, point.Data.Length);
Assert.AreEqual(174.68, point[0]);
Assert.AreEqual(26.87, point[1]);
Assert.AreEqual(39, (int)(point[2] / 1000000));
Assert.AreEqual(100, (int)(point[3] / 1000000));
itr.MoveNext();
point = itr.Current;
Assert.AreEqual(1, point.Sequence);
Assert.AreEqual(4, point.Data.Length);
Assert.AreEqual(168.18, point[0]);
Assert.AreEqual(25.88, point[1]);
Assert.AreEqual(29, (int)(point[2] / 1000000));
Assert.AreEqual(84, (int)(point[3] / 1000000));
itr.MoveNext();
point = itr.Current;
Assert.AreEqual(2, point.Sequence);
Assert.AreEqual(4, point.Data.Length);
Assert.AreEqual(170.12, point[0]);
Assert.AreEqual(25.98, point[1]);
Assert.AreEqual(18, (int)(point[2] / 1000000));
Assert.AreEqual(37, (int)(point[3] / 1000000));
// now check the actual data
Assert.AreEqual(16, marketData.Data.Count);
Assert.AreEqual(20, marketData.InputNeuronCount);
Assert.AreEqual(1, marketData.OutputNeuronCount);
IEnumerator <IMLDataPair> itr2 = marketData.Data.GetEnumerator();
itr2.MoveNext();
IMLDataPair pair = itr2.Current;
Assert.AreEqual(20, pair.Input.Count);
Assert.AreEqual(1, pair.Ideal.Count);
Assert.AreEqual(-0.037, Math.Round(pair.Input[0] * 1000.0) / 1000.0);
Assert.AreEqual(-0.037, Math.Round(pair.Input[1] * 1000.0) / 1000.0);
Assert.AreEqual(-0.246, Math.Round(pair.Input[2] * 1000.0) / 1000.0);
Assert.AreEqual(-0.156, Math.Round(pair.Input[3] * 1000.0) / 1000.0);
Assert.AreEqual(0.012, Math.Round(pair.Input[4] * 1000.0) / 1000.0);
Assert.AreEqual(0.0040, Math.Round(pair.Input[5] * 1000.0) / 1000.0);
Assert.AreEqual(-0.375, Math.Round(pair.Input[6] * 1000.0) / 1000.0);
Assert.AreEqual(-0.562, Math.Round(pair.Input[7] * 1000.0) / 1000.0);
Assert.AreEqual(0.03, Math.Round(pair.Input[8] * 1000.0) / 1000.0);
Assert.AreEqual(0.0020, Math.Round(pair.Input[9] * 1000.0) / 1000.0);
Assert.AreEqual(0.57, Math.Round(pair.Input[10] * 100.0) / 100.0);
Assert.AreEqual(0.929, Math.Round(pair.Input[11] * 1000.0) / 1000.0);
Assert.AreEqual(0.025, Math.Round(pair.Input[12] * 1000.0) / 1000.0);
Assert.AreEqual(-0.0070, Math.Round(pair.Input[13] * 1000.0) / 1000.0);
// for some reason, Yahoo likes to vary the volume numbers slightly, sometimes!
Assert.AreEqual(0.1, Math.Round(pair.Input[14] * 10.0) / 10.0);
Assert.AreEqual(-0.084, Math.Round(pair.Input[15] * 1000.0) / 1000.0);
Assert.AreEqual(-0.03, Math.Round(pair.Input[16] * 1000.0) / 1000.0);
Assert.AreEqual(-0.024, Math.Round(pair.Input[17] * 1000.0) / 1000.0);
Assert.AreEqual(0.008, Math.Round(pair.Input[18] * 1000.0) / 1000.0);
Assert.AreEqual(-0.172, Math.Round(pair.Input[19] * 1000.0) / 1000.0);
itr2.MoveNext();
pair = itr2.Current;
Assert.AreEqual(20, pair.Input.Count);
Assert.AreEqual(1, pair.Ideal.Count);
Assert.AreEqual(0.012, Math.Round(pair.Input[0] * 1000.0) / 1000.0);
Assert.AreEqual(0.0040, Math.Round(pair.Input[1] * 1000.0) / 1000.0);
Assert.AreEqual(-0.375, Math.Round(pair.Input[2] * 1000.0) / 1000.0);
Assert.AreEqual(-0.562, Math.Round(pair.Input[3] * 1000.0) / 1000.0);
Assert.AreEqual(0.03, Math.Round(pair.Input[4] * 1000.0) / 1000.0);
Assert.AreEqual(0.0020, Math.Round(pair.Input[5] * 1000.0) / 1000.0);
Assert.AreEqual(0.6, Math.Round(pair.Input[6] * 10.0) / 10.0);
Assert.AreEqual(0.929, Math.Round(pair.Input[7] * 1000.0) / 1000.0);
Assert.AreEqual(0.025, Math.Round(pair.Input[8] * 1000.0) / 1000.0);
Assert.AreEqual(-0.0070, Math.Round(pair.Input[9] * 1000.0) / 1000.0);
Assert.AreEqual(0.1, Math.Round(pair.Input[10] * 10.0) / 10.0);
Assert.AreEqual(-0.084, Math.Round(pair.Input[11] * 1000.0) / 1000.0);
Assert.AreEqual(-0.03, Math.Round(pair.Input[12] * 1000.0) / 1000.0);
Assert.AreEqual(-0.024, Math.Round(pair.Input[13] * 1000.0) / 1000.0);
Assert.AreEqual(0.0080, Math.Round(pair.Input[14] * 1000.0) / 1000.0);
Assert.AreEqual(-0.172, Math.Round(pair.Input[15] * 1000.0) / 1000.0);
Assert.AreEqual(0.014, Math.Round(pair.Input[16] * 1000.0) / 1000.0);
Assert.AreEqual(0.0090, Math.Round(pair.Input[17] * 1000.0) / 1000.0);
Assert.AreEqual(-0.1, Math.Round(pair.Input[18] * 10.0) / 10.0);
Assert.AreEqual(0.066, Math.Round(pair.Input[19] * 1000.0) / 1000.0);
}
public static double TrainNetworks(SupportVectorMachine network, MarketMLDataSet training)
{
// train the neural network
SVMTrain trainMain = new SVMTrain(network, training);
StopTrainingStrategy stop = new StopTrainingStrategy(0.0001, 200);
trainMain.AddStrategy(stop);
var sw = new Stopwatch();
sw.Start();
while (!stop.ShouldStop())
{
trainMain.PreIteration();
trainMain.Iteration();
trainMain.PostIteration();
Console.WriteLine(@"Iteration #:" + trainMain.IterationNumber + @" Error:" + trainMain.Error);
}
sw.Stop();
Console.WriteLine("SVM Trained in :" + sw.ElapsedMilliseconds + "For error:" + trainMain.Error + " Iterated:" + trainMain.IterationNumber);
return trainMain.Error;
}