static void FetchData()
{
var db = Database.GetProductionDatabase(MongoHostInfo.FromAppSettings());
var coll = db.MongoDatabase.GetCollection("DbBar");
coll.EnsureIndex(new IndexKeysBuilder().Ascending("Symbol", "Timestamp"));
var allBars = db.QueryAll <DbBar>(_ => true);
DateTime firstDate = DateTime.Parse("11/10/2001", null, DateTimeStyles.AdjustToUniversal);
DateTime lastDate = allBars.Any() ? allBars.OrderByDescending(x => x.Timestamp).First().Timestamp : firstDate;
var predicted = "DIA";
var firstFetchDate = lastDate.AddDays(1);
Console.WriteLine("Fetching from {0:MM/dd/yyyy} to today", firstFetchDate.Date);
VersaceDataFetching.DownloadData(predicted, firstFetchDate, DateTime.Now.Date);
Console.Write("Storing in mongo ...");
var allNewBars = new List <DbBar>();
foreach (var symbol in DataPreprocessing.GetTickers(predicted))
{
foreach (var dbBar in DataImport.LoadVersace(symbol).Select(x => new DbBar(db, symbol, x.Timestamp, x.Open, x.Low, x.High, x.Close, x.Volume)))
{
allNewBars.Add(dbBar);
}
}
db.StoreAll(allNewBars);
Console.WriteLine("done");
}
public static Mat TailorInputs(Mat inputMatrix, int databaseAInputLength, Chromosome chrom)
{
var inputs = inputMatrix.Columns();
// database selection
if (chrom.DatabaseType == DatabaseType.A)
{
inputs = inputs.Select(x => x.SubVector(0, databaseAInputLength)).ToList();
}
// complement coding
if (chrom.UseComplementCoding)
{
inputs = inputs.Select(DataPreprocessing.ComplementCode).ToList();
}
// PCA
if (chrom.UsePCA)
{
var principalComponents = DataPreprocessing.PrincipleComponents(inputs.ColumnsToMatrix());
var pcNumber = Math.Min(chrom.PrincipalComponent, principalComponents.ColumnCount - 1);
inputs = inputs.Select(x => DataPreprocessing.NthPrincipleComponent(principalComponents, pcNumber, x)).ToList();
}
return(inputs.ColumnsToMatrix());
}
static void DistributedEvolve(string masterRequestPath)
{
var masterReq = (MasterRequest)RabbitMessageReader.Read(0, File.ReadAllBytes(masterRequestPath));
PurgeTrainRequests();
using (var hostBroadcast = MakeBroadcaster())
{
Console.CancelKeyPress += (sender, eventArgs) => {
if (eventArgs.SpecialKey == ConsoleSpecialKey.ControlC)
{
hostBroadcast.Send(new HostStopEvolution());
}
if (eventArgs.SpecialKey == ConsoleSpecialKey.ControlBreak)
{
hostBroadcast.Send(new HostShutdown());
}
};
hostBroadcast.Send(new HostStartEvolution(masterReq));
var sw = new Stopwatch();
sw.Start();
var db = Database.GetProductionDatabase(MongoHostInfo.FromAppSettings());
var protoRun = db.QueryOne <ProtoRun>(x => x.Name == masterReq.ProtoRunName);
var dataSets = DataPreprocessing.LoadTrainingAndValidationSets(db, masterReq.Symbol, masterReq.StartDate.Date, masterReq.EndDate.Date,
masterReq.ValidationPct, GetSignalFunc(masterReq.SignalType));
Console.WriteLine("Data: {0:MM/dd/yyyy} - {1:MM/dd/yyyy}", masterReq.StartDate, masterReq.EndDate);
Console.WriteLine("Training set: {0} days", dataSets.Item1.Output.Count);
Console.WriteLine("Validation set: {0} days", dataSets.Item2.Output.Count);
var genSw = new Stopwatch();
genSw.Restart();
var run = Functions.Evolve(protoRun, new DistributedTrainer(), dataSets.Item1, dataSets.Item2,
masterReq.Symbol, masterReq.StartDate, masterReq.EndDate, masterReq.ValidationPct,
(genNum, completed, total) => Console.WriteLine("Generation {0}: Trained {1} of {2}", genNum, completed, total),
gen => {
Console.WriteLine("Gen {0} fitness {1} took {2}s, avg comp {3:N2}",
gen.Order, gen.Evaluated.Fitness, genSw.Elapsed.TotalSeconds, VectorSerializer.AverageCompressionRatio);
genSw.Restart();
});
hostBroadcast.Send(new HostStopEvolution());
Console.WriteLine("Finished Run {0} with fitness {1} in {2}", run.Id, run.Generations.Max(x => x.Evaluated.Fitness), sw.Elapsed);
Console.ReadKey();
}
}
static Tuple2 <DataSet> MakeTrainingAndValidationSets(Database db, string startDate, string endDate)
{
return(DataPreprocessing.LoadTrainingAndValidationSets(db, "DIA", DateTime.Parse(startDate, null, DateTimeStyles.AdjustToUniversal), DateTime.Parse(endDate, null, DateTimeStyles.AdjustToUniversal),
0.20, Signals.NextClose));
}
static DataSet MakeTrainingSet(Database db, string startDate, string endDate)
{
return(DataPreprocessing.LoadTrainingSet(db, "DIA", DateTime.Parse(startDate, null, DateTimeStyles.AdjustToUniversal), DateTime.Parse(endDate, null, DateTimeStyles.AdjustToUniversal),
Signals.NextClose));
}
public static DataSet GetData(string startDate, string endDate)
{
return(DataPreprocessing.LoadTrainingSetFromDisk("DIA", DateTime.Parse(startDate, null, DateTimeStyles.AdjustToUniversal), DateTime.Parse(endDate, null, DateTimeStyles.AdjustToUniversal),
Signals.NextClose));
}