protected double ScoreAI(MazeCycleOutcome mazeCycle, Traveler traveler)
{
const double GOOD_MOVE = 1.0;
const double BAD_MOVE = -1.0;
double retVal = BAD_MOVE;
Block block = gameRef.SimulationArea[traveler.location.X, traveler.location.Y];
//if (gameRef.BumpedObject)
//{
// Block block = gameRef.SimulationArea[traveler.location.X, traveler.location.Y];
// // empty block or good block or end block = good move
// if (block == null || (block != null && (block.Score > 0 || block.IsEndSimulation)))
// {
// retVal = GOOD_MOVE;
// }
//}
/* scores are either 1 or -1 */
if ((block == null && !mazeCycle.BumpedIntoWall) || (block != null && (block.Score > 0 || block.IsEndSimulation)))
{
retVal = GOOD_MOVE;
}
/* scores are either 1 or -1 BUT punishes if it goes back to previous block */
//if ((block == null && !mazeCycle.BumpedIntoWall) || (block != null && (block.Score > 0 || block.IsEndSimulation)))
//{
// if (previous != null && previous.X == traveler.location.X && previous.Y == traveler.location.Y)
// {
// retVal = BAD_MOVE;
// }
// else
// {
// retVal = GOOD_MOVE;
// }
//}
/* score is based on the block and 1 or -1 if empty or hits boundaries */
//if (block == null)
//{
// if (!mazeCycle.BumpedIntoWall)
// {
// retVal = GOOD_MOVE;
// }
//}
//else
//{
// retVal = block.Score;
//}
/* score is based on the block BUT punishes if it goes back to previous block */
//if (block == null)
//{
// if (!mazeCycle.BumpedIntoWall)
// {
// retVal = GOOD_MOVE;
// }
//}
//else
//{
// if (previous != null && previous.X == traveler.location.X && previous.Y == traveler.location.Y)
// {
// retVal = (block.Score > 0) ? -(block.Score) : block.Score;
// }
// else
// {
// retVal = block.Score;
// }
//}
previous = (mostRecent == null) ? null : new TravelerLocation()
{
X = mostRecent.X, Y = mostRecent.Y
};
mostRecent = new TravelerLocation()
{
X = traveler.location.X, Y = traveler.location.Y
};
return(retVal);
}
public virtual MazeCycleOutcome Travel(int?timerTimeout = null, CancellationToken?token = null)
{
const int START_RANDOM = 30;
const int END_RANDOM = 5;
OptimizationStatus?.Invoke("Initializing...");
int sessions = simSettings.SimType == SimulationType.Sessions ? simSettings.Sessions.Value : 100;
DateTime startedOn = simSettings.StartedOn = DateTime.Now;
if (simSettings.SimType == SimulationType.Time)
{
simSettings.EndsOn = startedOn.AddHours(simSettings.Hours.Value);
}
CreateOrLoadNetwork(simSettings.DBIdentifier, sessions, START_RANDOM, END_RANDOM);
OptimizationStatus?.Invoke("Training...");
MazeCycleOutcome outcome = null;
bool stopTraining = false;
do
{
for (int i = 0; i < sessions; i++)
{
outcome = Travel(true, token: token, perMoveDisplay: simSettings.EnableSimDisplay);
if ((simSettings.SimType == SimulationType.Score && outcome.FinalScore >= simSettings.Score.Value) ||
(simSettings.SimType == SimulationType.Time && simSettings.EndsOn < DateTime.Now) ||
(token.HasValue && token.Value.IsCancellationRequested))
{
stopTraining = true;
break;
}
}
if (simSettings.SimType == SimulationType.Sessions)
{
stopTraining = true;
}
ResetRLM();
} while (!stopTraining);
if (!token.HasValue || (token.HasValue && !token.Value.IsCancellationRequested))
{
if (simSettings.SimType == SimulationType.Score)
{
for (int i = 0; i < SimulationSettings.NUM_SCORE_HITS - 1; i++)
{
Travel(false, token: token, perMoveDisplay: simSettings.EnableSimDisplay);
}
}
Travel(false, timerTimeout, token, true);
TrainingDone();
OptimizationStatus?.Invoke("Done");
}
return(outcome);
}
private MazeCycleOutcome Travel(bool learn, int?timerTimeout = null, CancellationToken?token = null, bool perMoveDisplay = false)
{
IMazeGame game = GetNewGameInstance();
MazeCycleOutcome outcome = new MazeCycleOutcome();
// Start AI Training
var currentSessionID = network.SessionStart();
SessionStarted?.Invoke(network.RandomnessCurrentValue);
recentMoves.Clear();
//tmr.Interval = timerTimeout;
//tmr.Start();
//timeout = 0;
int movesCnt = 1;
while (!outcome.GameOver) // && timeout == 0)
{
if (token?.IsCancellationRequested == true)
{
break;
}
// set up input for x and y based on our current location on the maze
var inputs = new List <RlmIOWithValue>()
{
//new RlmIOWithValue(network.Inputs.First(a => a.Name == "X"), game.traveler.location.X.ToString()),
//new RlmIOWithValue(network.Inputs.First(a => a.Name == "Y"), game.traveler.location.Y.ToString()),
new RlmIOWithValue(network.Inputs.First(a => a.Name == "Move"), movesCnt.ToString())
};
// get AI output based on inputs
RlmCycle cycle = new RlmCycle();
var aiResult = cycle.RunCycle(network, currentSessionID, inputs, learn);
var direction = Convert.ToInt16(aiResult.CycleOutput.Outputs.First().Value);
// make the move
outcome = game.CycleMaze(direction);
// score AI output
double score = 0.0;
//score = ScoreAI(outcome, game.traveler);
network.ScoreCycle(aiResult.CycleOutput.CycleID, score);
if (timerTimeout.HasValue)
{
Task.Delay(timerTimeout.Value).Wait();
}
if (perMoveDisplay)
{
MazeCycleComplete?.Invoke(game.traveler.location.X, game.traveler.location.Y, outcome.BumpedIntoWall);
}
if (!learn)
{
recentMoves.Add(new MoveDetails()
{
Direction = direction, MoveNumber = movesCnt
});
}
movesCnt++;
}
//tmr.Stop();
// compute final score
outcome.FinalScore = game.CalculateFinalScore(game.Moves);
// End AI Training
network.SessionEnd(outcome.FinalScore);
SessionComplete?.Invoke(network.SessionCount, outcome.FinalScore, movesCnt - 1);
if (movesCnt > HighestMoveCount)
{
HighestMoveCount = movesCnt;
}
return(outcome);
}
