private void sARSALAMBDA(State currentState, Strategie strategie)
{
firstRound(currentState, strategie);
//Q(s,a)
QValue qValue = strategie.getQValue(lastState, action, qValues);
//e(s,a) <- e(s,a)+1
EValue eValue = strategie.getEValue(lastState, action, eValues);
//Stacking
//--> Replacing: eValue.setValue(1);
eValue.setValue(eValue.getValue() + 1);
// a ausführen
ExecuteAction(qValue.getAction());
// r beobachten
reward = getReward();
//s' beobachten
currentState = DetermineState();
//Q(s',a') gemäß Strategie
QValue currentQValue = strategie.getQValueForStrategie(currentState, qValues);
//Tabelle aktualisieren
qValues = SARSA_LAMBDA.updateQTable(lastState, lastQValue.getAction(), currentState, currentQValue.getAction(), reward, qValues, eValues);
//e(s,a) <- 𝛾𝜆e(s,a)
//updateETable --> eValue.setValue(gamma * lambda * eValue.getValue());
eValues = SARSA_LAMBDA.updateETable(lastState, lastQValue.getAction(), currentState, currentQValue.getAction(), eValues);
// s<-s', a<-a'
lastState = currentState;
lastQValue = currentQValue;
action = currentQValue.getAction();
}
private void sARSA(State currentState, Strategie strategie)
{
firstRound(currentState, strategie);
//Q(s,a)
QValue qValue = strategie.getQValue(lastState, action, qValues);
// a ausführen
ExecuteAction(qValue.getAction());
// r beobachten
reward = getReward();
//s' beobachten
currentState = DetermineState();
//Q(s',a') gemäß Strategie
QValue currentQValue = strategie.getQValueForStrategie(currentState, qValues);
//Tabelle aktualisieren
qValues = SARSA.updateTable(lastState, lastQValue.getAction(), currentState, currentQValue.getAction(), reward, qValues);
// s<-s', a<-a'
lastState = currentState;
lastQValue = currentQValue;
action = currentQValue.getAction();
}
private void firstRound(State currentState, Strategie strategie)
{
if (lastQValue.getAction().Equals(Assets.Scripts.Enum.Action.Begin))
{
lastState = currentState;
lastQValue = strategie.getQValueForStrategie(lastState, qValues);
action = lastQValue.getAction();
}
}
private void InitialiserStrategie()
{
Debug.Print("stratégie active");
Strategie.Ajouter(new ActionRobot(Homologation, () => true, () => 101, executionUnique: true));
Strategie.Ajouter(new ActionRobot(SortirZoneDepart, () => true, () => 100, executionUnique: true));
Strategie.Ajouter(new ActionRobot(RecupererCylindre1, () => true, () => 100, true));
Strategie.Ajouter(new ActionRobot(RecupererCylindresFusee, () => true, () => 99, true));
Strategie.Ajouter(new ActionRobot(RecupererCylindre2, () => true, () => 95, true));
Strategie.Ajouter(new ActionRobot(DeposerCylindres, () => true, () => 94, true));
}
private void qLearning(State currentState, Strategie strategie)
{
var qValue = strategie.getQValueForStrategie(currentState, qValues);
ExecuteAction(qValue.getAction());
if (!lastQValue.getAction().Equals(Assets.Scripts.Enum.Action.Begin))
{
qValues = QLearning.updateTable(lastQValue, qValue, qValues, getReward());
}
lastState = currentState;
lastQValue = qValue;
}
protected Veld MinimaxVeld(Bord bord)
{
Veld doel = null;
Strategie besteStrategie = Minimax(bord, Speler, 0);
int gekozenRij = besteStrategie.Veld.Rij;
int gekozenKolom = besteStrategie.Veld.Kolom;
foreach (Veld v in bord)
{
if (v.Kolom == gekozenKolom && v.Rij == gekozenRij)
{
doel = v;
}
}
return(doel);
}
// Form :
// [STRATEGY SAVE]
// <GRID>
// 0 0 0 0 ...
// </GRID>
// Spawn='... ...'
public string[] Save()
{
//Save strategy
string[] result = new string[5];
result[0] = Strategie.Save();
//Save grid
result[1] = "<GRID>";
String line = "";
for (int i = 0; i < nbSizeMap; i++)
{
for (int j = 0; j < nbSizeMap; j++)
{
Console.Write(i * nbSizeMap + j);
if (TileList[i * nbSizeMap + j].Equals(TileFactory.INSTANCE.TileDesert))
{
line += "0 ";
}
if (TileList[i * nbSizeMap + j].Equals(TileFactory.INSTANCE.TilePlain))
{
line += "1 ";
}
if (TileList[i * nbSizeMap + j].Equals(TileFactory.INSTANCE.TileSwamp))
{
line += "2 ";
}
if (TileList[i * nbSizeMap + j].Equals(TileFactory.INSTANCE.TileVolcano))
{
line += "3 ";
}
}
}
result [2] = line;
result [3] = "</GRID>";
result [4] = "Spawn='" + SpawnList[0] + " " + SpawnList[1] + "'";
return(result);
}
// Set the value as size, number of unit etc... in function of the strategy
public void SetStrategyValue()
{
nbPlayer = Strategie.GetNbPlayer();
nbSizeMap = Strategie.GetSizeMap();
nbUnitPerPlayer = Strategie.GetUnitPerPlayer();
}
public double GetDistance(Entity e, int pos)
{
int oldPos = e.Pos;
return((oldPos - 1 == pos || oldPos + 1 == pos || oldPos + Strategie.GetSizeMap() == pos || oldPos - Strategie.GetSizeMap() == pos)?e.Race.GetMovePointCost(this.GetTile(oldPos)):Double.MaxValue);
/*double[,] minValueTab = new double[nbSizeMap,nbSizeMap];
* int[,] precTab = new int[nbSizeMap , nbSizeMap];
*
* double minValueSwapped = 0;
*
* for (int i = 0; i < nbSizeMap; i++)
* {
* for(int j = 0; j < nbSizeMap; j++)
* {
* minValueTab[i,j] = Double.MaxValue;
* precTab[i,j] = -1;
* }
* }
* minValueTab[pos / nbSizeMap, pos % nbSizeMap] = 0;
* do
* {
* minValueSwapped = Double.MaxValue;
* for (int i = 0; i < nbSizeMap; i++)
* {
* for (int j = 0; j < nbSizeMap; j++)
* {
* if (precTab[i, j] != -1)
* {
* double myCost = minValueTab[i, j] + e.Race.GetMovePointCost(TileList[i * nbSizeMap + j]);
* if (j > 0)
* {
* if (minValueTab[i, j - 1] > myCost && (GetTeamOn(i*nbSizeMap+j-1)==0 || GetTeamOn(i*nbSizeMap+j-1)==e.Team) )
* {
* minValueTab[i, j - 1] = myCost;
* precTab[i, j - 1] = i * nbSizeMap + j;
* if (minValueSwapped > myCost)
* {
* minValueSwapped = myCost;
* }
* }
* }
* if (j < nbSizeMap - 1)
* {
* if (minValueTab[i, j + 1] > myCost && (GetTeamOn(i * nbSizeMap + j + 1) == 0 || GetTeamOn(i * nbSizeMap + j + 1) == e.Team))
* {
* minValueTab[i, j + 1] = myCost;
* precTab[i, j + 1] = i * nbSizeMap + j;
* if (minValueSwapped > myCost)
* {
* minValueSwapped = myCost;
* }
* }
* }
* if (i > 0)
* {
* if (minValueTab[i - 1, j] > myCost)
* {
* minValueTab[i - 1, j] = myCost;
* precTab[i - 1, j] = i * nbSizeMap + j;
* if (minValueSwapped > myCost && (GetTeamOn((i - 1) * nbSizeMap + j) == 0 || GetTeamOn((i - 1) * nbSizeMap + j) == e.Team))
* {
* minValueSwapped = myCost;
* }
* }
* }
* if (i < nbSizeMap - 1)
* {
* if (minValueTab[i + 1, j] > myCost)
* {
* minValueTab[i + 1, j] = myCost;
* precTab[i + 1, j] = i * nbSizeMap + j;
* if (minValueSwapped > myCost && (GetTeamOn((i + 1) * nbSizeMap + j) == 0 || GetTeamOn((i + 1) * nbSizeMap + j) == e.Team))
* {
* minValueSwapped = myCost;
* }
* }
* }
* }
* }
* }
* } while (minValueSwapped < precTab[pos / nbSizeMap, pos % nbSizeMap]);
* // Si aucun changement effectué : minValueSwapped = MAXVALUE > precTab[x,y]
* // Si minValueSwapped > precTab[x,y], aucun prochain changement ne fera baissé la valeur minimale du chemin.
* int lengthWay = 0;
* int prec = pos;
* while (precTab[prec / nbSizeMap, prec % nbSizeMap] != -1)
* {
* lengthWay++;
* prec = precTab[prec / nbSizeMap, prec % nbSizeMap];
* }
* double[] result = new double[lengthWay + 1];
* result[0] = minValueTab[pos / nbSizeMap, pos % nbSizeMap];
* prec = pos;
* while (precTab[prec / nbSizeMap, prec % nbSizeMap] != -1)
* {
* lengthWay--;
* result[lengthWay - 1] = prec;
* prec = precTab[prec / nbSizeMap, prec % nbSizeMap];
* }
* return result;*/
}
public override void JouerTour()
{
Strategie.Déplacer(this);
}
//Brute force
protected Strategie Minimax(Bord bord, Speler speler, int diepte)
{
var kopieBord = new Bord();
kopieBord = bord;
if (CheckOpWinst(kopieBord, Speler.O))
{
return(new Strategie(null, 100 + diepte));
}
else if (CheckOpWinst(kopieBord, Speler.X))
{
return(new Strategie(null, -100 - diepte));
}
else if (CheckOfVol(kopieBord))
{
return(new Strategie(null, 0));
}
else
{
diepte = diepte + 10;
if (speler == Speler.O)
{
Strategie besteStrategie = new Strategie(null, -100000);
foreach (Veld v in kopieBord)
{
if (v.IsVrij())
{
v.Speler = speler;
Strategie strategie = new Strategie(v, Minimax(kopieBord, Speler.X, diepte).Score);
if (strategie.Score > besteStrategie.Score)
{
besteStrategie = strategie;
}
//als er al een winnende move is ontdekt, hogere score toekennen als "broer" ook winst oplevert
else if (strategie.Score >= 100)
{
besteStrategie.Score = besteStrategie.Score + 50;
}
v.Speler = null;
}
}
return(besteStrategie);
}
else
{
Strategie besteStrategie = new Strategie(null, 100000);
foreach (Veld v in kopieBord)
{
if (v.IsVrij())
{
v.Speler = speler;
Strategie strategie = new Strategie(v, Minimax(kopieBord, Speler.O, diepte).Score);
if (strategie.Score < besteStrategie.Score)
{
besteStrategie = strategie;
}
else if (strategie.Score <= -100)
{
besteStrategie.Score = besteStrategie.Score - 50;
}
v.Speler = null;
}
}
return(besteStrategie);
}
}
}
