Ejemplo n.º 1
0
        static void FetchData()
        {
            var db   = Database.GetProductionDatabase(MongoHostInfo.FromAppSettings());
            var coll = db.MongoDatabase.GetCollection("DbBar");

            coll.EnsureIndex(new IndexKeysBuilder().Ascending("Symbol", "Timestamp"));
            var      allBars   = db.QueryAll <DbBar>(_ => true);
            DateTime firstDate = DateTime.Parse("11/10/2001", null, DateTimeStyles.AdjustToUniversal);
            DateTime lastDate  = allBars.Any() ? allBars.OrderByDescending(x => x.Timestamp).First().Timestamp : firstDate;
            var      predicted = "DIA";

            var firstFetchDate = lastDate.AddDays(1);

            Console.WriteLine("Fetching from {0:MM/dd/yyyy} to today", firstFetchDate.Date);

            VersaceDataFetching.DownloadData(predicted, firstFetchDate, DateTime.Now.Date);

            Console.Write("Storing in mongo ...");
            var allNewBars = new List <DbBar>();

            foreach (var symbol in DataPreprocessing.GetTickers(predicted))
            {
                foreach (var dbBar in DataImport.LoadVersace(symbol).Select(x => new DbBar(db, symbol, x.Timestamp, x.Open, x.Low, x.High, x.Close, x.Volume)))
                {
                    allNewBars.Add(dbBar);
                }
            }
            db.StoreAll(allNewBars);
            Console.WriteLine("done");
        }
Ejemplo n.º 2
0
        void Supervise()
        {
            ThreadSync = SyncContext.Current;

            Console.Write("Supervisor is loading dataset...");
            var db          = Database.GetProductionDatabase(MongoHostInfo.FromAppSettings());
            var trainingSet = DataPreprocessing.LoadTrainingAndValidationSets(db, MasterRequest.Symbol, MasterRequest.StartDate,
                                                                              MasterRequest.EndDate, MasterRequest.ValidationPct,
                                                                              VersaceMain.GetSignalFunc(MasterRequest.SignalType)).Item1;

            Console.WriteLine("done");

            Console.Write("Supervisor is starting {0} slaves..", SlaveCount);

            Slaves = Lists.Repeat(SlaveCount, _ => {
                Slave slave = new Slave(CloneDataSet(trainingSet));
                slave.Task  = Task.Factory.StartNew(slave.Run).ContinueWith(t => {
                    Slaves.RemoveAll(x => x.Task == t);
                    if (t.IsFaulted)
                    {
                        Console.WriteLine("Slave faulted: " + t.Exception);
                    }
                });
                return(slave);
            });

            Console.WriteLine("done");

            Waiter.Wait(() => !Slaves.Any());
        }
Ejemplo n.º 3
0
        static void DistributedEvolve(string masterRequestPath)
        {
            var masterReq = (MasterRequest)RabbitMessageReader.Read(0, File.ReadAllBytes(masterRequestPath));

            PurgeTrainRequests();

            using (var hostBroadcast = MakeBroadcaster())
            {
                Console.CancelKeyPress += (sender, eventArgs) => {
                    if (eventArgs.SpecialKey == ConsoleSpecialKey.ControlC)
                    {
                        hostBroadcast.Send(new HostStopEvolution());
                    }
                    if (eventArgs.SpecialKey == ConsoleSpecialKey.ControlBreak)
                    {
                        hostBroadcast.Send(new HostShutdown());
                    }
                };

                hostBroadcast.Send(new HostStartEvolution(masterReq));

                var sw = new Stopwatch();
                sw.Start();

                var db       = Database.GetProductionDatabase(MongoHostInfo.FromAppSettings());
                var protoRun = db.QueryOne <ProtoRun>(x => x.Name == masterReq.ProtoRunName);
                var dataSets = DataPreprocessing.LoadTrainingAndValidationSets(db, masterReq.Symbol, masterReq.StartDate.Date, masterReq.EndDate.Date,
                                                                               masterReq.ValidationPct, GetSignalFunc(masterReq.SignalType));

                Console.WriteLine("Data: {0:MM/dd/yyyy} - {1:MM/dd/yyyy}", masterReq.StartDate, masterReq.EndDate);
                Console.WriteLine("Training set: {0} days", dataSets.Item1.Output.Count);
                Console.WriteLine("Validation set: {0} days", dataSets.Item2.Output.Count);

                var genSw = new Stopwatch();
                genSw.Restart();
                var run = Functions.Evolve(protoRun, new DistributedTrainer(), dataSets.Item1, dataSets.Item2,
                                           masterReq.Symbol, masterReq.StartDate, masterReq.EndDate, masterReq.ValidationPct,
                                           (genNum, completed, total) => Console.WriteLine("Generation {0}: Trained {1} of {2}", genNum, completed, total),
                                           gen => {
                    Console.WriteLine("Gen {0} fitness {1} took {2}s, avg comp {3:N2}",
                                      gen.Order, gen.Evaluated.Fitness, genSw.Elapsed.TotalSeconds, VectorSerializer.AverageCompressionRatio);
                    genSw.Restart();
                });

                hostBroadcast.Send(new HostStopEvolution());

                Console.WriteLine("Finished Run {0} with fitness {1} in {2}", run.Id, run.Generations.Max(x => x.Evaluated.Fitness), sw.Elapsed);
                Console.ReadKey();
            }
        }
Ejemplo n.º 4
0
        public static Database GetProductionDatabase(MongoHostInfo mongo)
        {
            var mongoSettings = new MongoClientSettings {
                Server = new MongoServerAddress(mongo.Hostname)
            };

            if (mongo.Username != "guest")
            {
                mongoSettings.Credentials = new[] { MongoCredential.CreateMongoCRCredential(mongo.DatabaseName, mongo.Username, mongo.Password) }
            }
            ;
            var mongoClient = new MongoClient(mongoSettings);
            var mongoServer = mongoClient.GetServer();
            var mongoDb     = mongoServer.GetDatabase(mongo.DatabaseName);

            return(new Database(mongoDb));
        }

        void StoreInMongo <T>(T value) where T : MongoTopLevelObject
        {
            var coll = MDB.GetCollection(typeof(T).Name);

            coll.Insert(value);
            value.Database = this;
        }

        void StoreAllInMongo <T>(IEnumerable <T> values) where T : MongoTopLevelObject
        {
            var coll = MDB.GetCollection(typeof(T).Name);

            coll.InsertBatch(values);
            foreach (var value in values)
            {
                value.Database = this;
            }
        }
    }
Ejemplo n.º 5
0
 public Slave(DataSet trainingSet)
 {
     Database    = Database.GetProductionDatabase(MongoHostInfo.FromAppSettings());
     TrainingSet = trainingSet;
 }