Beispiel #1
0
        static void DistributedEvolve(string masterRequestPath)
        {
            var masterReq = (MasterRequest)RabbitMessageReader.Read(0, File.ReadAllBytes(masterRequestPath));

            PurgeTrainRequests();

            using (var hostBroadcast = MakeBroadcaster())
            {
                Console.CancelKeyPress += (sender, eventArgs) => {
                    if (eventArgs.SpecialKey == ConsoleSpecialKey.ControlC)
                    {
                        hostBroadcast.Send(new HostStopEvolution());
                    }
                    if (eventArgs.SpecialKey == ConsoleSpecialKey.ControlBreak)
                    {
                        hostBroadcast.Send(new HostShutdown());
                    }
                };

                hostBroadcast.Send(new HostStartEvolution(masterReq));

                var sw = new Stopwatch();
                sw.Start();

                var db       = Database.GetProductionDatabase(MongoHostInfo.FromAppSettings());
                var protoRun = db.QueryOne <ProtoRun>(x => x.Name == masterReq.ProtoRunName);
                var dataSets = DataPreprocessing.LoadTrainingAndValidationSets(db, masterReq.Symbol, masterReq.StartDate.Date, masterReq.EndDate.Date,
                                                                               masterReq.ValidationPct, GetSignalFunc(masterReq.SignalType));

                Console.WriteLine("Data: {0:MM/dd/yyyy} - {1:MM/dd/yyyy}", masterReq.StartDate, masterReq.EndDate);
                Console.WriteLine("Training set: {0} days", dataSets.Item1.Output.Count);
                Console.WriteLine("Validation set: {0} days", dataSets.Item2.Output.Count);

                var genSw = new Stopwatch();
                genSw.Restart();
                var run = Functions.Evolve(protoRun, new DistributedTrainer(), dataSets.Item1, dataSets.Item2,
                                           masterReq.Symbol, masterReq.StartDate, masterReq.EndDate, masterReq.ValidationPct,
                                           (genNum, completed, total) => Console.WriteLine("Generation {0}: Trained {1} of {2}", genNum, completed, total),
                                           gen => {
                    Console.WriteLine("Gen {0} fitness {1} took {2}s, avg comp {3:N2}",
                                      gen.Order, gen.Evaluated.Fitness, genSw.Elapsed.TotalSeconds, VectorSerializer.AverageCompressionRatio);
                    genSw.Restart();
                });

                hostBroadcast.Send(new HostStopEvolution());

                Console.WriteLine("Finished Run {0} with fitness {1} in {2}", run.Id, run.Generations.Max(x => x.Evaluated.Fitness), sw.Elapsed);
                Console.ReadKey();
            }
        }
Beispiel #2
0
 static void RunSupervisor()
 {
     while (true)
     {
         try
         {
             Console.WriteLine("Waiting for evolution to start");
             var hostStartMsg = Broadcaster.WaitFor <HostStartEvolution>();
             AppDomainIsolator.Run(hostStartMsg.MasterRequest.ToUtf8(), masterRequestBytes => {
                 Console.WriteLine("Built on " + new FileInfo(typeof(Functions).Assembly.Location).LastWriteTime);
                 RabbitMessageReader.Register(typeof(TrainRequest));
                 using (var bcast = MakeBroadcaster())
                 {
                     var nodeName = GetNodeName();
                     Console.WriteLine("HostUp   " + nodeName);
                     try
                     {
                         using (new Supervisor((int)(Environment.ProcessorCount /* * 1.5*/), (MasterRequest)RabbitMessageReader.Read(0, masterRequestBytes)))
                         {
                             Console.WriteLine("Waiting for evolution to stop");
                             bcast.WaitFor <HostStopEvolution>();
                             Console.WriteLine();
                             Console.WriteLine("Evolution stopped");
                         }
                     }
                     finally
                     {
                         Console.WriteLine("HostDown " + nodeName);
                         RNNInterop.FreeQuqeMathDll();
                     }
                 }
             });
         }
         catch (ShutdownException)
         {
             Console.WriteLine("Shutting down");
             return;
         }
     }
 }