static void FetchData()
{
var db = Database.GetProductionDatabase(MongoHostInfo.FromAppSettings());
var coll = db.MongoDatabase.GetCollection("DbBar");
coll.EnsureIndex(new IndexKeysBuilder().Ascending("Symbol", "Timestamp"));
var allBars = db.QueryAll <DbBar>(_ => true);
DateTime firstDate = DateTime.Parse("11/10/2001", null, DateTimeStyles.AdjustToUniversal);
DateTime lastDate = allBars.Any() ? allBars.OrderByDescending(x => x.Timestamp).First().Timestamp : firstDate;
var predicted = "DIA";
var firstFetchDate = lastDate.AddDays(1);
Console.WriteLine("Fetching from {0:MM/dd/yyyy} to today", firstFetchDate.Date);
VersaceDataFetching.DownloadData(predicted, firstFetchDate, DateTime.Now.Date);
Console.Write("Storing in mongo ...");
var allNewBars = new List <DbBar>();
foreach (var symbol in DataPreprocessing.GetTickers(predicted))
{
foreach (var dbBar in DataImport.LoadVersace(symbol).Select(x => new DbBar(db, symbol, x.Timestamp, x.Open, x.Low, x.High, x.Close, x.Volume)))
{
allNewBars.Add(dbBar);
}
}
db.StoreAll(allNewBars);
Console.WriteLine("done");
}
void Supervise()
{
ThreadSync = SyncContext.Current;
Console.Write("Supervisor is loading dataset...");
var db = Database.GetProductionDatabase(MongoHostInfo.FromAppSettings());
var trainingSet = DataPreprocessing.LoadTrainingAndValidationSets(db, MasterRequest.Symbol, MasterRequest.StartDate,
MasterRequest.EndDate, MasterRequest.ValidationPct,
VersaceMain.GetSignalFunc(MasterRequest.SignalType)).Item1;
Console.WriteLine("done");
Console.Write("Supervisor is starting {0} slaves..", SlaveCount);
Slaves = Lists.Repeat(SlaveCount, _ => {
Slave slave = new Slave(CloneDataSet(trainingSet));
slave.Task = Task.Factory.StartNew(slave.Run).ContinueWith(t => {
Slaves.RemoveAll(x => x.Task == t);
if (t.IsFaulted)
{
Console.WriteLine("Slave faulted: " + t.Exception);
}
});
return(slave);
});
Console.WriteLine("done");
Waiter.Wait(() => !Slaves.Any());
}
static void DistributedEvolve(string masterRequestPath)
{
var masterReq = (MasterRequest)RabbitMessageReader.Read(0, File.ReadAllBytes(masterRequestPath));
PurgeTrainRequests();
using (var hostBroadcast = MakeBroadcaster())
{
Console.CancelKeyPress += (sender, eventArgs) => {
if (eventArgs.SpecialKey == ConsoleSpecialKey.ControlC)
{
hostBroadcast.Send(new HostStopEvolution());
}
if (eventArgs.SpecialKey == ConsoleSpecialKey.ControlBreak)
{
hostBroadcast.Send(new HostShutdown());
}
};
hostBroadcast.Send(new HostStartEvolution(masterReq));
var sw = new Stopwatch();
sw.Start();
var db = Database.GetProductionDatabase(MongoHostInfo.FromAppSettings());
var protoRun = db.QueryOne <ProtoRun>(x => x.Name == masterReq.ProtoRunName);
var dataSets = DataPreprocessing.LoadTrainingAndValidationSets(db, masterReq.Symbol, masterReq.StartDate.Date, masterReq.EndDate.Date,
masterReq.ValidationPct, GetSignalFunc(masterReq.SignalType));
Console.WriteLine("Data: {0:MM/dd/yyyy} - {1:MM/dd/yyyy}", masterReq.StartDate, masterReq.EndDate);
Console.WriteLine("Training set: {0} days", dataSets.Item1.Output.Count);
Console.WriteLine("Validation set: {0} days", dataSets.Item2.Output.Count);
var genSw = new Stopwatch();
genSw.Restart();
var run = Functions.Evolve(protoRun, new DistributedTrainer(), dataSets.Item1, dataSets.Item2,
masterReq.Symbol, masterReq.StartDate, masterReq.EndDate, masterReq.ValidationPct,
(genNum, completed, total) => Console.WriteLine("Generation {0}: Trained {1} of {2}", genNum, completed, total),
gen => {
Console.WriteLine("Gen {0} fitness {1} took {2}s, avg comp {3:N2}",
gen.Order, gen.Evaluated.Fitness, genSw.Elapsed.TotalSeconds, VectorSerializer.AverageCompressionRatio);
genSw.Restart();
});
hostBroadcast.Send(new HostStopEvolution());
Console.WriteLine("Finished Run {0} with fitness {1} in {2}", run.Id, run.Generations.Max(x => x.Evaluated.Fitness), sw.Elapsed);
Console.ReadKey();
}
}
public static Database GetProductionDatabase(MongoHostInfo mongo)
{
var mongoSettings = new MongoClientSettings {
Server = new MongoServerAddress(mongo.Hostname)
};
if (mongo.Username != "guest")
{
mongoSettings.Credentials = new[] { MongoCredential.CreateMongoCRCredential(mongo.DatabaseName, mongo.Username, mongo.Password) }
}
;
var mongoClient = new MongoClient(mongoSettings);
var mongoServer = mongoClient.GetServer();
var mongoDb = mongoServer.GetDatabase(mongo.DatabaseName);
return(new Database(mongoDb));
}
void StoreInMongo <T>(T value) where T : MongoTopLevelObject
{
var coll = MDB.GetCollection(typeof(T).Name);
coll.Insert(value);
value.Database = this;
}
void StoreAllInMongo <T>(IEnumerable <T> values) where T : MongoTopLevelObject
{
var coll = MDB.GetCollection(typeof(T).Name);
coll.InsertBatch(values);
foreach (var value in values)
{
value.Database = this;
}
}
}
public Slave(DataSet trainingSet)
{
Database = Database.GetProductionDatabase(MongoHostInfo.FromAppSettings());
TrainingSet = trainingSet;
}