static void DistributedEvolve(string masterRequestPath)
{
var masterReq = (MasterRequest)RabbitMessageReader.Read(0, File.ReadAllBytes(masterRequestPath));
PurgeTrainRequests();
using (var hostBroadcast = MakeBroadcaster())
{
Console.CancelKeyPress += (sender, eventArgs) => {
if (eventArgs.SpecialKey == ConsoleSpecialKey.ControlC)
{
hostBroadcast.Send(new HostStopEvolution());
}
if (eventArgs.SpecialKey == ConsoleSpecialKey.ControlBreak)
{
hostBroadcast.Send(new HostShutdown());
}
};
hostBroadcast.Send(new HostStartEvolution(masterReq));
var sw = new Stopwatch();
sw.Start();
var db = Database.GetProductionDatabase(MongoHostInfo.FromAppSettings());
var protoRun = db.QueryOne <ProtoRun>(x => x.Name == masterReq.ProtoRunName);
var dataSets = DataPreprocessing.LoadTrainingAndValidationSets(db, masterReq.Symbol, masterReq.StartDate.Date, masterReq.EndDate.Date,
masterReq.ValidationPct, GetSignalFunc(masterReq.SignalType));
Console.WriteLine("Data: {0:MM/dd/yyyy} - {1:MM/dd/yyyy}", masterReq.StartDate, masterReq.EndDate);
Console.WriteLine("Training set: {0} days", dataSets.Item1.Output.Count);
Console.WriteLine("Validation set: {0} days", dataSets.Item2.Output.Count);
var genSw = new Stopwatch();
genSw.Restart();
var run = Functions.Evolve(protoRun, new DistributedTrainer(), dataSets.Item1, dataSets.Item2,
masterReq.Symbol, masterReq.StartDate, masterReq.EndDate, masterReq.ValidationPct,
(genNum, completed, total) => Console.WriteLine("Generation {0}: Trained {1} of {2}", genNum, completed, total),
gen => {
Console.WriteLine("Gen {0} fitness {1} took {2}s, avg comp {3:N2}",
gen.Order, gen.Evaluated.Fitness, genSw.Elapsed.TotalSeconds, VectorSerializer.AverageCompressionRatio);
genSw.Restart();
});
hostBroadcast.Send(new HostStopEvolution());
Console.WriteLine("Finished Run {0} with fitness {1} in {2}", run.Id, run.Generations.Max(x => x.Evaluated.Fitness), sw.Elapsed);
Console.ReadKey();
}
}
static void RunSupervisor()
{
while (true)
{
try
{
Console.WriteLine("Waiting for evolution to start");
var hostStartMsg = Broadcaster.WaitFor <HostStartEvolution>();
AppDomainIsolator.Run(hostStartMsg.MasterRequest.ToUtf8(), masterRequestBytes => {
Console.WriteLine("Built on " + new FileInfo(typeof(Functions).Assembly.Location).LastWriteTime);
RabbitMessageReader.Register(typeof(TrainRequest));
using (var bcast = MakeBroadcaster())
{
var nodeName = GetNodeName();
Console.WriteLine("HostUp " + nodeName);
try
{
using (new Supervisor((int)(Environment.ProcessorCount /* * 1.5*/), (MasterRequest)RabbitMessageReader.Read(0, masterRequestBytes)))
{
Console.WriteLine("Waiting for evolution to stop");
bcast.WaitFor <HostStopEvolution>();
Console.WriteLine();
Console.WriteLine("Evolution stopped");
}
}
finally
{
Console.WriteLine("HostDown " + nodeName);
RNNInterop.FreeQuqeMathDll();
}
}
});
}
catch (ShutdownException)
{
Console.WriteLine("Shutting down");
return;
}
}
}