A data set that is designed to hold market data. This class is based on the TemporalNeuralDataSet. This class is designed to load financial data from external sources. This class is designed to track financial data across days. However, it should be usable with other levels of granularity as well.
Наследование: Encog.ML.Data.Temporal.TemporalMLDataSet
Пример #1
0
        public void TestCSVLoader()
        {
            var loader = new CSVFinal();
            loader.DateFormat = "yyyy.MM.dd hh:mm:ss";

            var tickerAAPL = new TickerSymbol("AAPL", "NY");

            var desc = new MarketDataDescription(tickerAAPL, MarketDataType.Close, true, true);
            MarketMLDataSet marketData = new MarketMLDataSet(loader, 5, 1);
            marketData.AddDescription(desc);
            marketData.SequenceGrandularity = Util.Time.TimeUnit.Hours;
            var begin = new DateTime(2006, 1, 1);
            var end = new DateTime(2007, 7, 31);
            loader.GetFile((AssemblyDirectory + "\\smallCSV.csv"));
            marketData.Load(begin, end);
            marketData.Generate();
            // first test the points
            IEnumerator<TemporalPoint> itr = marketData.Points.GetEnumerator();
            itr.MoveNext();
            TemporalPoint point = itr.Current;

            Assert.AreEqual(0, point.Sequence);
            Assert.AreEqual(1, point.Data.Length);
            Assert.AreEqual(1.12884, point[0]);
            Assert.AreEqual(5, marketData.Points.Count);
        }
Пример #2
0
        public void TestCSVLoader()
        {
            var loader = new CSVFinal();

            loader.DateFormat = "yyyy.MM.dd hh:mm:ss";

            var tickerAAPL = new TickerSymbol("AAPL", "NY");

            var             desc       = new MarketDataDescription(tickerAAPL, MarketDataType.Close, true, true);
            MarketMLDataSet marketData = new MarketMLDataSet(loader, 5, 1);

            marketData.AddDescription(desc);
            marketData.SequenceGrandularity = Util.Time.TimeUnit.Hours;
            var begin = new DateTime(2006, 1, 1);
            var end   = new DateTime(2007, 7, 31);

            loader.GetFile((AssemblyDirectory + "\\smallCSV.csv"));
            marketData.Load(begin, end);
            marketData.Generate();
            // first test the points
            IEnumerator <TemporalPoint> itr = marketData.Points.GetEnumerator();

            itr.MoveNext();
            TemporalPoint point = itr.Current;

            Assert.AreEqual(0, point.Sequence);
            Assert.AreEqual(1, point.Data.Length);
            Assert.AreEqual(1.12884, point[0]);
            Assert.AreEqual(5, marketData.Points.Count);
        }
        public static void Generate(FileInfo dataDir)
        {
            IMarketLoader loader = new YahooFinanceLoader();
            var market = new MarketMLDataSet(loader,
                                             Config.INPUT_WINDOW, Config.PREDICT_WINDOW);
            var desc = new MarketDataDescription(
                Config.TICKER, MarketDataType.AdjustedClose, true, true);
            market.AddDescription(desc);

            var end = DateTime.Now; // end today
            var begin = new DateTime(end.Ticks); // begin 30 days ago

            // Gather training data for the last 2 years, stopping 60 days short of today.
            // The 60 days will be used to evaluate prediction.
            begin = begin.AddDays(-60);
            end = end.AddDays(-60);
            begin = begin.AddYears(-2);

            market.Load(begin, end);
            market.Generate();
            EncogUtility.SaveEGB(FileUtil.CombinePath(dataDir, Config.TRAINING_FILE), market);

            // create a network
            BasicNetwork network = EncogUtility.SimpleFeedForward(
                market.InputSize,
                Config.HIDDEN1_COUNT,
                Config.HIDDEN2_COUNT,
                market.IdealSize,
                true);

            // save the network and the training
            EncogDirectoryPersistence.SaveObject(FileUtil.CombinePath(dataDir, Config.NETWORK_FILE), network);
        }
        public static MarketMLDataSet GrabData()
        {
            IMarketLoader loader = new YahooFinanceLoader();
            var result = new MarketMLDataSet(loader,
                                             Config.INPUT_WINDOW, Config.PREDICT_WINDOW);
            var desc = new MarketDataDescription(Config.TICKER,
                                                 MarketDataType.AdjustedClose, true, true);
            result.AddDescription(desc);

            var end = DateTime.Now; // end today
            var begin = new DateTime(end.Ticks); // begin 30 days ago
            begin = begin.AddDays(-60);

            result.Load(begin, end);
            result.Generate();

            return result;
        }
        public static void Generate(string fileName)
        {

          
            FileInfo dataDir = new FileInfo(@Environment.CurrentDirectory);
            IMarketLoader loader = new CSVFinal();
            var market = new MarketMLDataSet(loader,CONFIG.INPUT_WINDOW, CONFIG.PREDICT_WINDOW);
          //  var desc = new MarketDataDescription(Config.TICKER, MarketDataType.Close, true, true);

            var desc = new MarketDataDescription(CONFIG.TICKER, MarketDataType.Close, true, true);
            market.AddDescription(desc);
            string currentDirectory =@"c:\";
            loader.GetFile(fileName);

            var end = DateTime.Now; // end today
            var begin = new DateTime(end.Ticks); // begin 30 days ago

            // Gather training data for the last 2 years, stopping 60 days short of today.
            // The 60 days will be used to evaluate prediction.
            begin = begin.AddDays(-600);
            end = begin.AddDays(200);
           
            Console.WriteLine("You are loading date from:" + begin.ToShortDateString() + " To :" + end.ToShortDateString());

            market.Load(begin, end);
            market.Generate();
            EncogUtility.SaveEGB(FileUtil.CombinePath(dataDir, CONFIG.SVMTRAINING_FILE), market);

            // create a network
            //BasicNetwork network = EncogUtility.SimpleFeedForward(
            //    market.InputSize,
            //    CONFIG.HIDDEN1_COUNT,
            //    CONFIG.HIDDEN2_COUNT,
            //    market.IdealSize,
            //    true);


            SupportVectorMachine network = new SupportVectorMachine(CONFIG.INPUT_WINDOW, true);
            TrainNetworks(network, market);
            // save the network and the training
            EncogDirectoryPersistence.SaveObject(FileUtil.CombinePath(dataDir,CONFIG.SVMTRAINING_FILE), network);
        }
Пример #6
0
        public static MarketMLDataSet GrabData(string newfileLoad)
        {
            IMarketLoader loader = new CSVLoader();
            loader.GetFile(newfileLoad);

            var result = new MarketMLDataSet(loader,
                                             Config.INPUT_WINDOW, Config.PREDICT_WINDOW);
          //  var desc = new MarketDataDescription(Config.TICKER,
                                              //   MarketDataType.Close, true, true);

            var desc = new MarketDataDescription(Config.TICKER,
                                     MarketDataType.Trade, true, true);
            result.AddDescription(desc);

            var end = DateTime.Now; // end today
            var begin = new DateTime(end.Ticks); // begin 30 days ago
            begin = begin.AddDays(-150);

            result.Load(begin, end);
            result.Generate();

            return result;
        }
Пример #7
0
        public void MarketData()
        {
            IMarketLoader loader = new YahooFinanceLoader();
            var tickerAAPL = new TickerSymbol("AAPL", null);
            var tickerMSFT = new TickerSymbol("MSFT", null);
            var marketData = new MarketMLDataSet(loader, 5, 1);
            marketData.AddDescription(new MarketDataDescription(tickerAAPL, MarketDataType.Close, true, true));
            marketData.AddDescription(new MarketDataDescription(tickerMSFT, MarketDataType.Close, true, false));
            marketData.AddDescription(new MarketDataDescription(tickerAAPL, MarketDataType.Volume, true, false));
            marketData.AddDescription(new MarketDataDescription(tickerMSFT, MarketDataType.Volume, true, false));
            var begin = new DateTime(2008, 7, 1);
            var end = new DateTime(2008, 7, 31);
            marketData.Load(begin, end);
            marketData.Generate();
            Assert.AreEqual(22, marketData.Points.Count);

            // first test the points
            IEnumerator<TemporalPoint> itr = marketData.Points.GetEnumerator();
            itr.MoveNext();
            TemporalPoint point = itr.Current;

            Assert.AreEqual(0, point.Sequence);
            Assert.AreEqual(4, point.Data.Length);
            Assert.AreEqual(174.68, point[0]);
            Assert.AreEqual(26.87, point[1]);
            Assert.AreEqual(39, (int) (point[2]/1000000));
            Assert.AreEqual(100, (int) (point[3]/1000000));

            itr.MoveNext();
            point = itr.Current;
            Assert.AreEqual(1, point.Sequence);
            Assert.AreEqual(4, point.Data.Length);
            Assert.AreEqual(168.18, point[0]);
            Assert.AreEqual(25.88, point[1]);
            Assert.AreEqual(29, (int) (point[2]/1000000));
            Assert.AreEqual(84, (int) (point[3]/1000000));

            itr.MoveNext();
            point = itr.Current;
            Assert.AreEqual(2, point.Sequence);
            Assert.AreEqual(4, point.Data.Length);
            Assert.AreEqual(170.12, point[0]);
            Assert.AreEqual(25.98, point[1]);
            Assert.AreEqual(18, (int) (point[2]/1000000));
            Assert.AreEqual(37, (int) (point[3]/1000000));

            // now check the actual data
            Assert.AreEqual(16, marketData.Data.Count);
            Assert.AreEqual(20, marketData.InputNeuronCount);
            Assert.AreEqual(1, marketData.OutputNeuronCount);

            IEnumerator<IMLDataPair> itr2 = marketData.Data.GetEnumerator();
            itr2.MoveNext();
            IMLDataPair pair = itr2.Current;
            Assert.AreEqual(20, pair.Input.Count);
            Assert.AreEqual(1, pair.Ideal.Count);

            Assert.AreEqual(-0.037, Math.Round(pair.Input[0]*1000.0)/1000.0);
            Assert.AreEqual(-0.037, Math.Round(pair.Input[1]*1000.0)/1000.0);
            Assert.AreEqual(-0.246, Math.Round(pair.Input[2]*1000.0)/1000.0);
            Assert.AreEqual(-0.156, Math.Round(pair.Input[3]*1000.0)/1000.0);
            Assert.AreEqual(0.012, Math.Round(pair.Input[4]*1000.0)/1000.0);
            Assert.AreEqual(0.0040, Math.Round(pair.Input[5]*1000.0)/1000.0);
            Assert.AreEqual(-0.375, Math.Round(pair.Input[6]*1000.0)/1000.0);
            Assert.AreEqual(-0.562, Math.Round(pair.Input[7]*1000.0)/1000.0);
            Assert.AreEqual(0.03, Math.Round(pair.Input[8]*1000.0)/1000.0);
            Assert.AreEqual(0.0020, Math.Round(pair.Input[9]*1000.0)/1000.0);
            Assert.AreEqual(0.57, Math.Round(pair.Input[10]*100.0)/100.0);
            Assert.AreEqual(0.929, Math.Round(pair.Input[11]*1000.0)/1000.0);
            Assert.AreEqual(0.025, Math.Round(pair.Input[12]*1000.0)/1000.0);
            Assert.AreEqual(-0.0070, Math.Round(pair.Input[13]*1000.0)/1000.0);
            // for some reason, Yahoo likes to vary the volume numbers slightly, sometimes!
            Assert.AreEqual(0.1, Math.Round(pair.Input[14]*10.0)/10.0);
            Assert.AreEqual(-0.084, Math.Round(pair.Input[15]*1000.0)/1000.0);
            Assert.AreEqual(-0.03, Math.Round(pair.Input[16]*1000.0)/1000.0);
            Assert.AreEqual(-0.024, Math.Round(pair.Input[17]*1000.0)/1000.0);
            Assert.AreEqual(0.008, Math.Round(pair.Input[18]*1000.0)/1000.0);
            Assert.AreEqual(-0.172, Math.Round(pair.Input[19]*1000.0)/1000.0);

            itr2.MoveNext();
            pair = itr2.Current;
            Assert.AreEqual(20, pair.Input.Count);
            Assert.AreEqual(1, pair.Ideal.Count);

            Assert.AreEqual(0.012, Math.Round(pair.Input[0]*1000.0)/1000.0);
            Assert.AreEqual(0.0040, Math.Round(pair.Input[1]*1000.0)/1000.0);
            Assert.AreEqual(-0.375, Math.Round(pair.Input[2]*1000.0)/1000.0);
            Assert.AreEqual(-0.562, Math.Round(pair.Input[3]*1000.0)/1000.0);
            Assert.AreEqual(0.03, Math.Round(pair.Input[4]*1000.0)/1000.0);
            Assert.AreEqual(0.0020, Math.Round(pair.Input[5]*1000.0)/1000.0);
            Assert.AreEqual(0.6, Math.Round(pair.Input[6]*10.0)/10.0);
            Assert.AreEqual(0.929, Math.Round(pair.Input[7]*1000.0)/1000.0);
            Assert.AreEqual(0.025, Math.Round(pair.Input[8]*1000.0)/1000.0);
            Assert.AreEqual(-0.0070, Math.Round(pair.Input[9]*1000.0)/1000.0);
            Assert.AreEqual(0.1, Math.Round(pair.Input[10]*10.0)/10.0);
            Assert.AreEqual(-0.084, Math.Round(pair.Input[11]*1000.0)/1000.0);
            Assert.AreEqual(-0.03, Math.Round(pair.Input[12]*1000.0)/1000.0);
            Assert.AreEqual(-0.024, Math.Round(pair.Input[13]*1000.0)/1000.0);
            Assert.AreEqual(0.0080, Math.Round(pair.Input[14]*1000.0)/1000.0);
            Assert.AreEqual(-0.172, Math.Round(pair.Input[15]*1000.0)/1000.0);
            Assert.AreEqual(0.014, Math.Round(pair.Input[16]*1000.0)/1000.0);
            Assert.AreEqual(0.0090, Math.Round(pair.Input[17]*1000.0)/1000.0);
            Assert.AreEqual(-0.1, Math.Round(pair.Input[18]*10.0)/10.0);
            Assert.AreEqual(0.066, Math.Round(pair.Input[19]*1000.0)/1000.0);
        }
Пример #8
0
        public void MarketData()
        {
            IMarketLoader loader     = new YahooFinanceLoader();
            var           tickerAAPL = new TickerSymbol("AAPL", null);
            var           tickerMSFT = new TickerSymbol("MSFT", null);
            var           marketData = new MarketMLDataSet(loader, 5, 1);

            marketData.AddDescription(new MarketDataDescription(tickerAAPL, MarketDataType.Close, true, true));
            marketData.AddDescription(new MarketDataDescription(tickerMSFT, MarketDataType.Close, true, false));
            marketData.AddDescription(new MarketDataDescription(tickerAAPL, MarketDataType.Volume, true, false));
            marketData.AddDescription(new MarketDataDescription(tickerMSFT, MarketDataType.Volume, true, false));
            var begin = new DateTime(2008, 7, 1);
            var end   = new DateTime(2008, 7, 31);

            marketData.Load(begin, end);
            marketData.Generate();
            Assert.AreEqual(22, marketData.Points.Count);

            // first test the points
            IEnumerator <TemporalPoint> itr = marketData.Points.GetEnumerator();

            itr.MoveNext();
            TemporalPoint point = itr.Current;

            Assert.AreEqual(0, point.Sequence);
            Assert.AreEqual(4, point.Data.Length);
            Assert.AreEqual(174.68, point[0]);
            Assert.AreEqual(26.87, point[1]);
            Assert.AreEqual(39, (int)(point[2] / 1000000));
            Assert.AreEqual(100, (int)(point[3] / 1000000));

            itr.MoveNext();
            point = itr.Current;
            Assert.AreEqual(1, point.Sequence);
            Assert.AreEqual(4, point.Data.Length);
            Assert.AreEqual(168.18, point[0]);
            Assert.AreEqual(25.88, point[1]);
            Assert.AreEqual(29, (int)(point[2] / 1000000));
            Assert.AreEqual(84, (int)(point[3] / 1000000));

            itr.MoveNext();
            point = itr.Current;
            Assert.AreEqual(2, point.Sequence);
            Assert.AreEqual(4, point.Data.Length);
            Assert.AreEqual(170.12, point[0]);
            Assert.AreEqual(25.98, point[1]);
            Assert.AreEqual(18, (int)(point[2] / 1000000));
            Assert.AreEqual(37, (int)(point[3] / 1000000));

            // now check the actual data
            Assert.AreEqual(16, marketData.Data.Count);
            Assert.AreEqual(20, marketData.InputNeuronCount);
            Assert.AreEqual(1, marketData.OutputNeuronCount);

            IEnumerator <IMLDataPair> itr2 = marketData.Data.GetEnumerator();

            itr2.MoveNext();
            IMLDataPair pair = itr2.Current;

            Assert.AreEqual(20, pair.Input.Count);
            Assert.AreEqual(1, pair.Ideal.Count);

            Assert.AreEqual(-0.037, Math.Round(pair.Input[0] * 1000.0) / 1000.0);
            Assert.AreEqual(-0.037, Math.Round(pair.Input[1] * 1000.0) / 1000.0);
            Assert.AreEqual(-0.246, Math.Round(pair.Input[2] * 1000.0) / 1000.0);
            Assert.AreEqual(-0.156, Math.Round(pair.Input[3] * 1000.0) / 1000.0);
            Assert.AreEqual(0.012, Math.Round(pair.Input[4] * 1000.0) / 1000.0);
            Assert.AreEqual(0.0040, Math.Round(pair.Input[5] * 1000.0) / 1000.0);
            Assert.AreEqual(-0.375, Math.Round(pair.Input[6] * 1000.0) / 1000.0);
            Assert.AreEqual(-0.562, Math.Round(pair.Input[7] * 1000.0) / 1000.0);
            Assert.AreEqual(0.03, Math.Round(pair.Input[8] * 1000.0) / 1000.0);
            Assert.AreEqual(0.0020, Math.Round(pair.Input[9] * 1000.0) / 1000.0);
            Assert.AreEqual(0.57, Math.Round(pair.Input[10] * 100.0) / 100.0);
            Assert.AreEqual(0.929, Math.Round(pair.Input[11] * 1000.0) / 1000.0);
            Assert.AreEqual(0.025, Math.Round(pair.Input[12] * 1000.0) / 1000.0);
            Assert.AreEqual(-0.0070, Math.Round(pair.Input[13] * 1000.0) / 1000.0);
            // for some reason, Yahoo likes to vary the volume numbers slightly, sometimes!
            Assert.AreEqual(0.1, Math.Round(pair.Input[14] * 10.0) / 10.0);
            Assert.AreEqual(-0.084, Math.Round(pair.Input[15] * 1000.0) / 1000.0);
            Assert.AreEqual(-0.03, Math.Round(pair.Input[16] * 1000.0) / 1000.0);
            Assert.AreEqual(-0.024, Math.Round(pair.Input[17] * 1000.0) / 1000.0);
            Assert.AreEqual(0.008, Math.Round(pair.Input[18] * 1000.0) / 1000.0);
            Assert.AreEqual(-0.172, Math.Round(pair.Input[19] * 1000.0) / 1000.0);

            itr2.MoveNext();
            pair = itr2.Current;
            Assert.AreEqual(20, pair.Input.Count);
            Assert.AreEqual(1, pair.Ideal.Count);

            Assert.AreEqual(0.012, Math.Round(pair.Input[0] * 1000.0) / 1000.0);
            Assert.AreEqual(0.0040, Math.Round(pair.Input[1] * 1000.0) / 1000.0);
            Assert.AreEqual(-0.375, Math.Round(pair.Input[2] * 1000.0) / 1000.0);
            Assert.AreEqual(-0.562, Math.Round(pair.Input[3] * 1000.0) / 1000.0);
            Assert.AreEqual(0.03, Math.Round(pair.Input[4] * 1000.0) / 1000.0);
            Assert.AreEqual(0.0020, Math.Round(pair.Input[5] * 1000.0) / 1000.0);
            Assert.AreEqual(0.6, Math.Round(pair.Input[6] * 10.0) / 10.0);
            Assert.AreEqual(0.929, Math.Round(pair.Input[7] * 1000.0) / 1000.0);
            Assert.AreEqual(0.025, Math.Round(pair.Input[8] * 1000.0) / 1000.0);
            Assert.AreEqual(-0.0070, Math.Round(pair.Input[9] * 1000.0) / 1000.0);
            Assert.AreEqual(0.1, Math.Round(pair.Input[10] * 10.0) / 10.0);
            Assert.AreEqual(-0.084, Math.Round(pair.Input[11] * 1000.0) / 1000.0);
            Assert.AreEqual(-0.03, Math.Round(pair.Input[12] * 1000.0) / 1000.0);
            Assert.AreEqual(-0.024, Math.Round(pair.Input[13] * 1000.0) / 1000.0);
            Assert.AreEqual(0.0080, Math.Round(pair.Input[14] * 1000.0) / 1000.0);
            Assert.AreEqual(-0.172, Math.Round(pair.Input[15] * 1000.0) / 1000.0);
            Assert.AreEqual(0.014, Math.Round(pair.Input[16] * 1000.0) / 1000.0);
            Assert.AreEqual(0.0090, Math.Round(pair.Input[17] * 1000.0) / 1000.0);
            Assert.AreEqual(-0.1, Math.Round(pair.Input[18] * 10.0) / 10.0);
            Assert.AreEqual(0.066, Math.Round(pair.Input[19] * 1000.0) / 1000.0);
        }
        public static double TrainNetworks(SupportVectorMachine network, MarketMLDataSet training)
        {
            // train the neural network
            SVMTrain trainMain = new SVMTrain(network, training);

            StopTrainingStrategy stop = new StopTrainingStrategy(0.0001, 200);
            trainMain.AddStrategy(stop);


            var sw = new Stopwatch();
            sw.Start();
            while (!stop.ShouldStop())
            {
                trainMain.PreIteration();


                trainMain.Iteration();
                trainMain.PostIteration();

                Console.WriteLine(@"Iteration #:" + trainMain.IterationNumber + @" Error:" + trainMain.Error);
            }
            sw.Stop();
            Console.WriteLine("SVM Trained in :" + sw.ElapsedMilliseconds + "For error:" + trainMain.Error + " Iterated:" + trainMain.IterationNumber);
            return trainMain.Error;
        }