private void __Run()
{
int EPOCHS = SolverConfig.GetInstance().epochs;
Console.WriteLine("Level: " + SolverConfig.GetInstance().level);
Console.WriteLine("Epochs: " + EPOCHS);
IExploration exp = new EGreedyExploration(SolverConfig.GetInstance().epsilon, 0f);
exp.Init(0f, 0f);
//tester.Start(new BoltzmannExploration(2, 0, -4));
_solver.Start(_tester);
//BasePool.StrongControl = true;
for (int e = 0; e < EPOCHS; e++)
{
while (_solver.Started)
{
_solver.Update();
}
_solver.Reset();
if (!_tester)
{
_solver.UpdateParams(e, EPOCHS);
}
Console.WriteLine(exp.ToString());
//BasePool.Instance.Check();
}
BasePool.Instance.Check();
Quit();
}
override public void Init(NeuralNetwork p_network = null)
{
NeuralNetwork network = null;
Optimizer optimizer = null;
if (p_network == null)
{
network = new NeuralNetwork();
network.AddLayer("input", new InputLayer(GetParam(STATE_DIM)), BaseLayer.TYPE.INPUT);
network.AddLayer("hidden0", new CoreLayer(SolverConfig.GetInstance().hidden_layer, ACTIVATION.RELU, BaseLayer.TYPE.HIDDEN), BaseLayer.TYPE.HIDDEN);
network.AddLayer("output", new CoreLayer(GetParam(ACTION_DIM), ACTIVATION.TANH, BaseLayer.TYPE.OUTPUT), BaseLayer.TYPE.OUTPUT);
// feed-forward connections
network.AddConnection("input", "hidden0", Connection.INIT.GLOROT_UNIFORM);
network.AddConnection("hidden0", "output", Connection.INIT.GLOROT_UNIFORM);
}
else
{
network = p_network;
}
optimizer = new ADAM(network);
//optimizer = new RMSProp(network);
//optimizer = new BackProp(network, 1e-5f, 0.99f, true);
_critic = new DeepQLearning(optimizer, network, 0.99f, SolverConfig.GetInstance().memory_size, SolverConfig.GetInstance().batch_size, SolverConfig.GetInstance().qtupdate_size);
_critic.SetAlpha(SolverConfig.GetInstance().learning_rate);
}
public void Start()
{
for (int i = 0; i < SolverConfig.GetInstance().async_learners; i++)
{
_learners[i].Start(i);
}
}
override public void Init(NeuralNetwork p_network = null)
{
_qtUpdateIndex = 0;
_qtUpdateSize = SolverConfig.GetInstance().qtupdate_size;
_networkQ = null;
if (p_network == null)
{
_networkQ = new NeuralNetwork();
_networkQ.AddLayer("input", new InputLayer(GetParam(STATE_DIM)), BaseLayer.TYPE.INPUT);
_networkQ.AddLayer("hidden0", new CoreLayer(SolverConfig.GetInstance().hidden_layer, ACTIVATION.RELU, BaseLayer.TYPE.HIDDEN), BaseLayer.TYPE.HIDDEN);
_networkQ.AddLayer("output", new CoreLayer(GetParam(ACTION_DIM), ACTIVATION.LINEAR, BaseLayer.TYPE.OUTPUT), BaseLayer.TYPE.OUTPUT);
// feed-forward connections
_networkQ.AddConnection("input", "hidden0", Connection.INIT.GLOROT_UNIFORM);
_networkQ.AddConnection("hidden0", "output", Connection.INIT.GLOROT_UNIFORM);
}
else
{
_networkQ = p_network;
}
CreateNetworkQt();
for (int i = 0; i < _learners.Capacity; i++)
{
AsyncDoubleQLearning worker = new AsyncDoubleQLearning(new ADAM(_networkQ), _networkQ, _networkQt, 0.99f, SolverConfig.GetInstance().async_update);
//worker.SetAlpha(SolverConfig.GetInstance().learning_rate);
worker.Optimizer.InitAlpha(SolverConfig.GetInstance().learning_rate, SolverConfig.GetInstance().learning_rate / 10);
_learners.Add(worker);
}
}
public SolverManager()
{
FG = 0;
_timestamp = DateTime.Now.ToFileTime();
_logger = new Logger();
#if !DEBUG
_logger.Init(".\\app_" + _timestamp + ".log");
#endif
_logger.Log(JsonConvert.SerializeObject(SolverConfig.GetInstance()));
int asyncLearners = SolverConfig.GetInstance().async_learners;
_actorMonitor = new Dictionary <int, int>();
_inputInterface = new InputInterface();
_encoder = new StateEncoder(_inputInterface);
_agent = new AsyncQN(asyncLearners);
_learners = new AsyncSolver[asyncLearners];
for (int i = 0; i < asyncLearners; i++)
{
IExploration exp = null;
exp = new BoltzmannExploration(SolverConfig.GetInstance().epsilon, 0.13f);
exp.Init(0.13f);
//exp = new EGreedyExploration(SolverConfig.GetInstance().epsilon, 0f);
//exp.Init(0.05f, 0f);
_learners[i] = new AsyncSolver(_inputInterface, _encoder, _agent, exp, _logger);
}
}
public void Run()
{
int EPOCHS = SolverConfig.GetInstance().epochs;
Console.WriteLine("Epochs: " + EPOCHS);
MazeExample tester = new MazeExample();
//BasePool.StrongControl = true;
IExploration exp = new EGreedyExploration(SolverConfig.GetInstance().epsilon, 0f);
exp.Init(0.02f, 0f);
//IExploration exp = new BoltzmannExploration(0.12f, 0.06f);
tester.Start(exp);
for (int e = 0; e < EPOCHS; e++)
{
while (tester.Started)
{
tester.Update();
}
tester.Reset();
tester.UpdateParams(e, EPOCHS);
//BasePool.Instance.Check();
}
BasePool.Instance.Check();
tester.Quit();
}
public void Update(int p_epochs)
{
for (int i = 0; i < SolverConfig.GetInstance().async_learners; i++)
{
if (_learners[i].Started)
{
_learners[i].Update();
}
else
{
FG++;
_learners[i].Reset();
_learners[i].UpdateParams(FG, p_epochs);
if (_actorMonitor.ContainsKey(i))
{
_actorMonitor[i]++;
}
else
{
_actorMonitor[i] = 1;
}
foreach (int v in _actorMonitor.Values)
{
Console.Write(v + " ");
}
Console.WriteLine();
Console.WriteLine("Epoch " + FG);
}
_agent.UpdateQt();
}
}
public void Start(bool p_test = false)
{
_test = p_test;
_wins = _loses = 0;
_DEBUG_logger = new Logger();
_DEBUG_limit = SolverConfig.GetInstance().batch_size * 15;
Reset();
}
public void Init(Action p_initialized)
{
_initialized = p_initialized;
int seed = SolverConfig.GetInstance().seed;
/*
* if (seed == 0)
* {
* seed = RandomGenerator.getInstance().rand(1, 5);
* }
*/
(new StartGameRequest(_inputInterface)).Send(SolverConfig.GetInstance().level, seed, OnStartGameCompleted);
}
public void Reset()
{
_accReward = 0;
waitingForInterface = true; // musi byt este pred send
_level = SolverConfig.GetInstance().level;
_seed = SolverConfig.GetInstance().seed;
if (_seed == 0)
{
_seed = RandomGenerator.getInstance().Rand(1, 5);
}
(new StartGameRequest(_inputInterface)).Send(SolverConfig.GetInstance().level, _seed, OnStartGameCompleted);
_DEBUG_clicks = new List <Vector>();
started = true;
}
private void __Run()
{
int EPOCHS = SolverConfig.GetInstance().epochs;
Console.WriteLine("Parallelism: " + SolverConfig.GetInstance().async_learners);
Console.WriteLine("Level: " + SolverConfig.GetInstance().level);
Console.WriteLine("Epochs: " + EPOCHS);
Console.WriteLine("Epoch " + _solver.FG);
_solver.Start();
while (_solver.FG < EPOCHS)
{
_solver.Update(EPOCHS);
}
BasePool.Instance.Check();
_solver.Quit();
}
public Experiment1()
{
SolverConfig.GetInstance().Load(@".\config.json");
_timestamp = DateTime.Now.ToFileTime();
_logger = new Logger();
#if !DEBUG
_logger.Init(".\\app_" + _timestamp + ".log");
#endif
_logger.Log(JsonConvert.SerializeObject(SolverConfig.GetInstance()));
int asyncLearners = SolverConfig.GetInstance().async_learners;
_inputInterface = new InputInterface();
_encoder = new StateEncoder(_inputInterface);
_agent = new DDQN();
IExploration exp = new EGreedyExploration(SolverConfig.GetInstance().epsilon, 0f);
exp.Init(0.1f, 1f);
_solver = new Solver(_inputInterface, _encoder, _agent, exp, _logger);
}
private void OnStartGameCompleted(StartGameRequest.ResultData result)
{
_encoder.PrepareEncoderProjection(result.levelConfig);
_agent.SetParam(BaseAgent.ACTION_DIM, _encoder.GetActionDim());
_agent.SetParam(BaseAgent.BOOST_INDEX, _encoder.GetActionDim() - 1);
_agent.SetParam(BaseAgent.STATE_DIM, _encoder.GetStateDim());
Console.WriteLine("Max. moves: " + result.levelConfig.moves);
Console.WriteLine("Action dim: " + _agent.GetParam(BaseAgent.ACTION_DIM));
Console.WriteLine("State dim: " + _agent.GetParam(BaseAgent.STATE_DIM));
if (SolverConfig.GetInstance().filename != string.Empty)
{
string filename = ".\\" + SolverConfig.GetInstance().filename;
Console.WriteLine("Loading file " + filename);
_agent.Load(filename);
Console.WriteLine(filename + " loaded.");
}
else
{
_agent.Init();
}
_initialized.Invoke();
}
public Experiment2()
{
SolverConfig.GetInstance().Load(@".\config.json");
_solver = new SolverManager();
}
public Example1()
{
SolverConfig.GetInstance().Load(@".\config.maze.json");
}