public void AddErrorAndOutLogAndWriteTwoLinesToEach() { var fileNameErr = MethodInfo.GetCurrentMethod().Name + "_Err.log"; var fileNameOut = MethodInfo.GetCurrentMethod().Name + "_Out.log"; var output = new Output(); Assert.IsNotNull(output); output.AddLog(new Log(fileNameErr, true, false)); output.AddLog(new Log(fileNameOut, true, false)); Assert.AreEqual(output.NumLogs, 2); Assert.IsTrue(File.Exists(fileNameErr)); Assert.IsTrue(File.Exists(fileNameOut)); output.GetLog(0).Writer.WriteLine("Hello Error!"); output.GetLog(0).Writer.WriteLine("Hello Error again!"); output.GetLog(1).Writer.WriteLine("Hello Out!"); output.GetLog(1).Writer.WriteLine("Hello Out again!"); output.Close(); using (var reader = new StreamReader(new FileStream(fileNameErr, FileMode.Open, FileAccess.Read, FileShare.Read))) { var line = reader.ReadLine(); Assert.AreEqual(line, "Hello Error!"); line = reader.ReadLine(); Assert.AreEqual(line, "Hello Error again!"); } using (var reader = new StreamReader(new FileStream(fileNameOut, FileMode.Open, FileAccess.Read, FileShare.Read))) { var line = reader.ReadLine(); Assert.AreEqual(line, "Hello Out!"); line = reader.ReadLine(); Assert.AreEqual(line, "Hello Out again!"); } }
public void WarningToConsoleErrorAndOutputLog() { var output = new Output(); Assert.IsNotNull(output); output.AddLog(Log.D_STDERR, true); output.AddLog(Log.D_STDOUT, true); output.Warning("Ooops!"); Assert.AreEqual(output.NumLogs, 2); output.Close(); }
public void ErrorToConsoleErrorAndOutputLogWithOneParameterToBlame() { var output = new Output(); Assert.IsNotNull(output); output.AddLog(Log.D_STDERR, true); output.AddLog(Log.D_STDOUT, true); output.Error("Ooops!", new Parameter("BadParameter")); Assert.AreEqual(output.NumLogs, 2); output.Close(); }
public void WarningToConsoleErrorAndOutputLogWithTwoParametersToBlame() { var output = new Output(); Assert.IsNotNull(output); output.AddLog(Log.D_STDERR, true); output.AddLog(Log.D_STDOUT, true); output.Warning("Ooops!", new Parameter("FirstBadParameter"), new Parameter("SecondBadParameter")); Assert.AreEqual(output.NumLogs, 2); output.Close(); }
public void MessageToStandardErrorAndOutputStreams() { var output = new Output(); output.AddLog(Log.D_STDERR, true); output.AddLog(Log.D_STDOUT, true); output.Message("Message!"); Assert.AreEqual(output.NumAnnouncements, 2); // one for each registered log context.WriteLine("NumAnnouncements = {0}", output.NumAnnouncements); output.Close(); }
public void GetNumLogs() { var output = new Output(); Assert.AreEqual(output.NumLogs, 0); output.AddLog(0, true); Assert.AreEqual(output.NumLogs, 1); output.AddLog(1, false); Assert.AreEqual(output.NumLogs, 2); output.Close(); }
public void FatalToConsoleErrorAndOutputLog() { var output = new Output(); Assert.IsNotNull(output); // Setting ThrowsErrors to true ensures that we don't exit the process! output.ThrowsErrors = true; output.AddLog(Log.D_STDERR, true); output.AddLog(Log.D_STDOUT, true); output.Fatal("Ooops!"); Assert.AreEqual(output.NumLogs, 2); output.Close(); }
public void FatalToConsoleErrorAndOutputLogWithTwoParametersToBlame() { var output = new Output(); Assert.IsNotNull(output); // Setting ThrowsErrors to true ensures that we don't exit the process! output.ThrowsErrors = true; output.AddLog(Log.D_STDERR, true); output.AddLog(Log.D_STDOUT, true); output.Fatal("Ooops!", new Parameter("FirstBadParameter"), new Parameter("SecondBadParameter")); Assert.AreEqual(output.NumLogs, 2); output.Close(); }
public void FilePrefixWithMultipleNumberedLogs() { var prefix = "Output_"; var output = new Output { FilePrefix = prefix }; output.AddLog("1.log"); Assert.IsTrue(File.Exists(prefix + "1.log")); output.AddLog("2.log"); Assert.IsTrue(File.Exists(prefix + "2.log")); output.Close(); }
public void FilePrefixWithRepeatNameThrows() { var prefix = "Output_"; var output = new Output { FilePrefix = prefix }; output.AddLog("1.log"); Assert.IsTrue(File.Exists(prefix + "1.log")); output.AddLog("1.log"); Assert.IsTrue(File.Exists(prefix + "1.log")); output.Close(); }
/// <summary> /// Initializes an evolutionary run given the parameters and a random seed adjustment (added to each random seed). /// The adjustment offers a convenient way to change the seeds of the random number generators each time you /// do a new evolutionary run. You are of course welcome to replace the random number generators after initialize(...) /// but before startFresh(...) /// </summary> public static EvolutionState Initialize(IParameterDatabase parameters, int randomSeedOffset) { var output = new Output(true); // stdout is always log #0. stderr is always log #1. // stderr accepts announcements, and both are fully verbose // by default. output.AddLog(Log.D_STDOUT, false); output.AddLog(Log.D_STDERR, true); // now continue intialization return(Initialize(parameters, randomSeedOffset, output)); }
/// <summary> /// Constructs and sets up an Output object. /// </summary> public static Output BuildOutput() { // 1. create the output var output = new Output(true); // stdout is always log #0. stderr is always log #1. // stderr accepts announcements, and both are fully verbose // by default. output.AddLog(Log.D_STDOUT, false); output.AddLog(Log.D_STDERR, true); return(output); }
public void AddErrorLog() { var output = new Output(); Assert.IsNotNull(output); output.AddLog(new Log("err.log", true, false)); Assert.AreEqual(output.NumLogs, 1); Assert.IsTrue(File.Exists("err.log")); }
public void AddConsoleOutLogUsingDescriptor() { var output = new Output(); Assert.IsNotNull(output); output.AddLog(Log.D_STDOUT, true); Assert.AreEqual(output.NumLogs, 1); output.Close(); }
public void FilePrefixDotOnlyThrows() { var prefix = "."; var output = new Output { FilePrefix = prefix }; output.AddLog(""); Assert.IsTrue(File.Exists(prefix + "")); output.Close(); }
public void FilePrefixUnderscoreOnly() { var prefix = "_"; var output = new Output { FilePrefix = prefix }; output.AddLog(""); Assert.IsTrue(File.Exists(prefix + "")); output.Close(); }
public void FilePrefixDotOnlyExtension() { var prefix = "."; var output = new Output { FilePrefix = prefix }; output.AddLog(".log"); Assert.IsTrue(File.Exists(prefix + ".log")); output.Close(); }
public void ErrorsNotedAndCleared() { var output = new Output(); output.AddLog(Log.D_STDERR, true); output.Error("Oops!"); Assert.IsTrue(output.HasErrors); output.ClearErrors(); Assert.IsFalse(output.HasErrors); output.Close(); }
public void FilePrefixNumericUnderscore() { var prefix = "1_"; var output = new Output { FilePrefix = prefix }; output.AddLog("test.log"); Assert.IsTrue(File.Exists(prefix + "test.log")); output.Close(); }
public void PrintLnToConsoleOut() { var output = new Output(); var logNum = output.AddLog(Log.D_STDOUT, true); output.PrintLn("Hello World!", logNum); output.PrintLn("Hello Again World!", logNum); // Prints on same line output.PrintLn("", logNum); // Now we've moved to a new line. output.PrintLn("Hello for a third time World!", logNum); output.PrintLn("Hello for a fourth time World!", logNum); output.Close(); }
public void StoreAnnouncementsDisableAndClear() { var output = new Output(); output.AddLog(Log.D_STDERR, true); output.Message("Hello World!"); Assert.AreEqual(output.NumAnnouncements, 1); output.StoreAnnouncements = false; Assert.AreEqual(output.NumAnnouncements, 1); output.ClearAnnouncements(); Assert.AreEqual(output.NumAnnouncements, 0); output.Message("Hi again!"); Assert.AreEqual(output.NumAnnouncements, 0); output.Close(); }
public void AddErrorLogAndWriteThenTryReadWithoutClosingOuputThrows() { var fileNameErr = MethodInfo.GetCurrentMethod().Name + "_Err.log"; var output = new Output(); Assert.IsNotNull(output); output.AddLog(new Log(fileNameErr, true, false)); Assert.AreEqual(output.NumLogs, 1); Assert.IsTrue(File.Exists(fileNameErr)); output.GetLog(0).Writer.WriteLine("Hello World!"); var reader = new StreamReader(new FileStream(fileNameErr, FileMode.Open, FileAccess.Read, FileShare.Read)); var line = reader.ReadLine(); Assert.AreEqual(line, "Hello World!"); }
public void Evaluate(IEvolutionState state, Individual ind, int subpopulation, int threadnum) { if (ind.Evaluated && !ReevaluateIndividuals) { return; } var fits = new ArrayList(); Individual bestOfRuns = null; for (int run = 0; run < Runs; run++) { // too annoying //state.Output.message("Thread " + threadnum + " Run " + run); try { // The following uses BinaryFormatter to create a separate copy of the ParameterDatabase. CurrentDatabase = (ParameterDatabase)p_database.DeepClone(); } catch (Exception e) { state.Output.Fatal("Exception copying database.\n" + e); } ModifyParameters(state, CurrentDatabase, run, ind); var output = new Output(false); // do not store messages, just print them output.AddLog(Log.D_STDOUT, false); output.AddLog(Log.D_STDERR, true); output.ThrowsErrors = true; // don't do System.exit(1); EvolutionState evaluatedState = null; try { evaluatedState = Evolve.Initialize(CurrentDatabase, 0, output); // should we override the seeds? if (SetRandom) { // we use the random number generator to seed the generators // of the underlying process. This isn't optimal but it should // probably do okay. To be extra careful we prime the generators. for (int i = 0; i < evaluatedState.Random.Length; i++) { int seed = state.Random[threadnum].NextInt(); evaluatedState.Random[i] = Evolve.PrimeGenerator(new MersenneTwisterFast(seed)); } } evaluatedState.Run(EvolutionState.C_STARTED_FRESH); // Issue a warning if there's more than one subpopulation if (evaluatedState.Population.Subpops.Count > 1) { state.Output.WarnOnce( "MetaProblem used, but underlying evolution state has more than one subpopulation: only the results from subpopulation 0 will be considered."); } // Identify the best fitness of the underlying EvolutionState run, // we can only easily detect if the underlying EvolutionState has a proper Statistics // object we can use AFTER we've run it because the Statistics object is set up during // run(). We could modify this but I'm too lazy to do so, so... Individual[] inds = null; // will get set, don't worry if (evaluatedState.Statistics != null && (evaluatedState.Statistics is SimpleStatistics || evaluatedState.Statistics is SimpleShortStatistics)) { inds = null; // obviously we need an interface here rather than this nonsense if (evaluatedState.Statistics is SimpleStatistics) { inds = ((SimpleStatistics)evaluatedState.Statistics).GetBestSoFar(); } else { inds = ((SimpleShortStatistics)evaluatedState.Statistics).GetBestSoFar(); } if (inds == null) { state.Output.Fatal( "Underlying evolution state has a Statistics object which provides a null best-so-far array. Can't extract fitness."); } fits.Add((Fitness)(inds[0].Fitness)); //System.err.println("" + inds[0] + " " + inds[0].fitness); } else if (evaluatedState.Statistics == null) { state.Output.Fatal( "Underlying evolution state has a null Statistics object. Can't extract fitness."); } else { state.Output.Fatal( "Underlying evolution state has a Statistics object which doesn't implement ProvidesBestSoFar. Can't extract fitness."); } // Now we need to suck out the best individual discovered so far. If the underlying // evoluationary system itself has a MetaProblem, we need to do this recursively. // We presume that the MetaProblem exists in subpopulation 0. if (evaluatedState.Evaluator.p_problem is MetaProblem) { var mp = (MetaProblem)evaluatedState.Evaluator.p_problem; lock (mp.Lock) { Individual bestind = mp.BestUnderlyingIndividual[0]; if (bestOfRuns == null || bestind.Fitness.BetterThan(bestOfRuns.Fitness)) { bestOfRuns = (Individual)bestind.Clone(); } } } // otherwise we grab the best individual found in the underlying evolutionary run, // gathered from the inds array we used earlier. else { // gather the best individual found during the runs if (bestOfRuns == null || inds[0].Fitness.BetterThan(bestOfRuns.Fitness)) { bestOfRuns = (Individual)(inds[0].Clone()); } } // now clean up Evolve.Cleanup(evaluatedState); } catch (OutputExitException e) { // looks like an error occurred. state.Output.Warning( "Error occurred in underlying evolutionary run. NOTE: multiple threads may still be running:\n" + e.Message); } catch (OutOfMemoryException e) { // Let's try fixing things evaluatedState = null; //System.gc(); GC.Collect(); GC.WaitForPendingFinalizers(); GC.Collect(); GC.WaitForPendingFinalizers(); state.Output.Warning( "An Out of Memory error occurred in underlying evolutionary run. Attempting to recover and reset. NOTE: multiple threads may still be running:\n" + e.Message); } } // Load the fitness into our individual var fits2 = new IFitness[fits.Count]; for (var i = 0; i < fits2.Length; i++) { fits2[i] = (IFitness)fits[i]; } Combine(state, fits2, ind.Fitness); ind.Evaluated = true; // store the best individual found during the runs if it's superior. // We need to do a lock here, which is rare in ECJ. This is because the // bestUnderlyingIndividual array is shared among MetaProblem instances lock (Lock) { if (bestOfRuns != null && (BestUnderlyingIndividual[subpopulation] == null || bestOfRuns.Fitness.BetterThan(BestUnderlyingIndividual[subpopulation].Fitness))) { BestUnderlyingIndividual[subpopulation] = bestOfRuns; // no clone necessary } } }
public static void Main(string[] args) { IEvolutionState state = null; IParameterDatabase parameters = null; Output output = null; //bool store; int x; // 0. find the parameter database for (x = 0; x < args.Length - 1; x++) { if (args[x].Equals(A_FILE)) { try { parameters = new ParameterDatabase(new FileInfo(new FileInfo(args[x + 1]).FullName), args); // add the fact that I am a slave: eval.i-am-slave = true // this is used only by the Evaluator to determine whether to use the MasterProblem parameters.SetParameter(new Parameter(EvolutionState.P_EVALUATOR).Push(Evaluator.P_IAMSLAVE), "true"); break; } catch (FileNotFoundException e) { Output.InitialError( "A File Not Found Exception was generated upon" + " reading the parameter file \"" + args[x + 1] + "\".\nHere it is:\n" + e, false); Environment.Exit(1); // This was originally part of the InitialError call in ECJ. But we make Slave responsible. } catch (IOException e) { Output.InitialError("An IO Exception was generated upon reading the" + " parameter file \"" + args[x + 1] + "\".\nHere it is:\n" + e, false); Environment.Exit(1); // This was originally part of the InitialError call in ECJ. But we make Slave responsible. } } } if (parameters == null) { Output.InitialError("No parameter file was specified.", false); Environment.Exit(1); // This was originally part of the InitialError call in ECJ. But we make Slave responsible. } // 5. Determine whether or not to return entire Individuals or just Fitnesses // (plus whether or not the Individual has been evaluated). var returnIndividuals = parameters.GetBoolean(new Parameter(P_RETURNINDIVIDUALS), null, false); // 5.5 should we silence the whole thing? bool silent = parameters.GetBoolean(new Parameter(P_SILENT), null, false); if (parameters.ParameterExists(new Parameter(P_MUZZLE), null)) { Output.InitialWarning("" + new Parameter(P_MUZZLE) + " has been deprecated. We suggest you use " + new Parameter(P_SILENT) + " or similar newer options."); } silent = silent || parameters.GetBoolean(new Parameter(P_MUZZLE), null, false); // 6. Open a server socket and listen for requests var slaveName = parameters.GetString(new Parameter(P_EVALSLAVENAME), null); var masterHost = parameters.GetString(new Parameter(P_EVALMASTERHOST), null); if (masterHost == null) { Output.InitialError("Master Host missing", new Parameter(P_EVALMASTERHOST)); } var masterPort = parameters.GetInt(new Parameter(P_EVALMASTERPORT), null, 0); if (masterPort == -1) { Output.InitialError("Master Port missing", new Parameter(P_EVALMASTERPORT)); } var useCompression = parameters.GetBoolean(new Parameter(P_EVALCOMPRESSION), null, false); RunTime = parameters.GetInt(new Parameter(P_RUNTIME), null, 0); RunEvolve = parameters.GetBoolean(new Parameter(P_RUNEVOLVE), null, false); OneShot = parameters.GetBoolean(new Parameter(P_ONESHOT), null, true); if (RunEvolve && !returnIndividuals) { Output.InitialError( "You have the slave running in 'evolve' mode, but it's only returning fitnesses to the master, not whole individuals. This is almost certainly wrong.", new Parameter(P_RUNEVOLVE), new Parameter(P_RETURNINDIVIDUALS)); Environment.Exit(1); // This was originally part of the InitialError call in ECJ. But we make Slave responsible. } if (!silent) { Output.InitialMessage("ECCS Slave"); if (RunEvolve) { Output.InitialMessage("Running in Evolve mode, evolve time is " + RunTime + " milliseconds"); } if (returnIndividuals) { Output.InitialMessage("Whole individuals will be returned"); } else { Output.InitialMessage("Only fitnesses will be returned"); } } // Continue to serve new masters until killed. TcpClient socket = null; // BRS: TcpClient is a wrapper around the Socket class while (true) { try { long connectAttemptCount = 0; if (!silent) { Output.InitialMessage("Connecting to master at " + masterHost + ":" + masterPort); } while (true) { try { socket = new TcpClient(masterHost, masterPort); break; } catch (Exception) // it's not up yet... { connectAttemptCount++; try { Thread.Sleep(new TimeSpan((Int64)10000 * SLEEP_TIME)); } catch (ThreadInterruptedException) { } } } if (!silent) { Output.InitialMessage("Connected to master after " + (connectAttemptCount * SLEEP_TIME) + " ms"); } BinaryReader dataIn = null; BinaryWriter dataOut = null; try { Stream tmpIn = socket.GetStream(); Stream tmpOut = socket.GetStream(); if (useCompression) { //Output.InitialError("JDK 1.5 has broken compression. For now, you must set eval.compression=false"); //Environment.Exit(1); // This was originally part of the InitialError call in ECJ. But we make Slave responsible. /* * tmpIn = new CompressingInputStream(tmpIn); * tmpOut = new CompressingOutputStream(tmpOut); */ tmpIn = Output.MakeCompressingInputStream(tmpIn); tmpOut = Output.MakeCompressingOutputStream(tmpOut); if (tmpIn == null || tmpOut == null) { var err = "You do not appear to have JZLib installed on your system, and so must set eval.compression=false. " + "To get JZLib, download from the ECJ website or from http://www.jcraft.com/jzlib/"; if (!silent) { Output.InitialMessage(err); } throw new OutputExitException(err); } } dataIn = new BinaryReader(tmpIn); dataOut = new BinaryWriter(tmpOut); } catch (IOException e) { var err = "Unable to open input stream from socket:\n" + e; if (!silent) { Output.InitialMessage(err); } throw new OutputExitException(err); } // specify the slaveName if (slaveName == null) { // BRS : TODO : Check equivalence of the address returned from .NET socket.Client.LocalEndPoint slaveName = socket.Client.LocalEndPoint + "/" + (DateTime.Now.Ticks - 621355968000000000) / 10000; if (!silent) { Output.InitialMessage("No slave name specified. Using: " + slaveName); } } dataOut.Write(slaveName); // Default encoding of BinaryWriter is UTF-8 dataOut.Flush(); // 1. create the output // store = parameters.GetBoolean(new Parameter(P_STORE), null, false); if (output != null) { output.Close(); } output = new Output(storeAnnouncementsInMemory: false) // do not store messages, just print them { ThrowsErrors = true // don't do System.exit(1) }; // stdout is always log #0. stderr is always log #1. // stderr accepts announcements, and both are fully verbose by default. output.AddLog(Log.D_STDOUT, false); output.AddLog(Log.D_STDERR, true); if (silent) { output.GetLog(0).Silent = true; output.GetLog(1).Silent = true; } if (!silent) { output.SystemMessage(ECVersion.Message()); } // 2. set up thread values int breedthreads = Evolve.DetermineThreads(output, parameters, new Parameter(Evolve.P_BREEDTHREADS)); int evalthreads = Evolve.DetermineThreads(output, parameters, new Parameter(Evolve.P_EVALTHREADS)); // Note that either breedthreads or evalthreads (or both) may be 'auto'. We don't warn about this because // the user isn't providing the thread seeds. // 3. create the Mersenne Twister random number generators, one per thread var random = new IMersenneTwister[breedthreads > evalthreads ? breedthreads : evalthreads]; var seed = dataIn.ReadInt32(); for (var i = 0; i < random.Length; i++) { random[i] = Evolve.PrimeGenerator(new MersenneTwisterFast(seed++)); } // we prime the generator to be more sure of randomness. // 4. Set up the evolution state // what evolution state to use? state = (IEvolutionState) parameters.GetInstanceForParameter(new Parameter(Evolve.P_STATE), null, typeof(IEvolutionState)); state.Parameters = new ParameterDatabase(); state.Parameters.AddParent(parameters); state.Random = random; state.Output = output; state.EvalThreads = evalthreads; state.BreedThreads = breedthreads; state.Setup(state, null); state.Population = state.Initializer.SetupPopulation(state, 0); // 5. Optionally do further loading var storage = state.Evaluator.MasterProblem; storage.ReceiveAdditionalData(state, dataIn); storage.TransferAdditionalData(state); try { while (true) { var newState = state; if (RunEvolve) { // Construct and use a new EvolutionState. This will be inefficient the first time around // as we've set up TWO EvolutionStates in a row with no good reason. IParameterDatabase coverDatabase = new ParameterDatabase(); // protect the underlying one coverDatabase.AddParent(state.Parameters); newState = Evolve.Initialize(coverDatabase, 0); newState.StartFresh(); newState.Output.Message("Replacing random number generators, ignore above seed message"); newState.Random = state.Random; // continue with RNG storage.TransferAdditionalData(newState); // load the arbitrary data again } // 0 means to shut down //Console.Error.WriteLine("reading next problem"); int problemType = dataIn.ReadByte(); //Console.Error.WriteLine("Read problem: " + problemType); switch (problemType) { case (int)SlaveEvaluationType.Shutdown: socket.Close(); if (OneShot) { return; // we're outa here } else { throw new OutputExitException("SHUTDOWN"); } case (int)SlaveEvaluationType.Simple: EvaluateSimpleProblem(newState, returnIndividuals, dataIn, dataOut, args); break; case (int)SlaveEvaluationType.Grouped: EvaluateGroupedProblem(newState, returnIndividuals, dataIn, dataOut); break; default: state.Output.Fatal("Unknown problem form specified: " + problemType); break; } //System.err.PrintLn("Done Evaluating Individual"); } } catch (IOException e) { // Since an IOException can happen here if the peer closes the socket // on it's end, we don't necessarily have to exit. Maybe we don't // even need to print a warning, but we'll do so just to indicate // something happened. state.Output.Fatal( "Unable to read type of evaluation from master. Maybe the master closed its socket and exited?:\n" + e); } catch (Exception e) { if (state != null) { state.Output.Fatal(e.Message); } else if (!silent) { Console.Error.WriteLine("FATAL ERROR (EvolutionState not created yet): " + e.Message); } } } catch (OutputExitException e) { // here we restart if necessary try { socket.Close(); } catch (Exception e2) { } if (OneShot) { Environment.Exit(0); } } catch (OutOfMemoryException e) { // Let's try fixing things state = null; GC.Collect(); GC.WaitForPendingFinalizers(); GC.Collect(); GC.WaitForPendingFinalizers(); try { socket.Close(); } catch (Exception e2) { } socket = null; // TODO: Overkill? Track memory before and after. GC.Collect(); GC.WaitForPendingFinalizers(); GC.Collect(); GC.WaitForPendingFinalizers(); Console.Error.WriteLine(e); if (OneShot) { Environment.Exit(0); } } if (!silent) { Output.InitialMessage("\n\nResetting Slave"); } } }