public ParetoTreeNode(Playfield state, ParetoTreeNode parent, int childIndex, RandomRoller roller,
ParetoTreePolicy treePolicy, Random rnd, ParetoMCTSPlayer a_player)
{
this.m_player = a_player;
this.state = state;
this.parent = parent;
this.m_rnd = rnd;
//children = new ParetoTreeNode[ParetoMCTSParameters.NUM_ACTIONS];
this.roller = roller;
this.mTreePolicy = treePolicy;
pa = new ParetoArchive();
this.childIndex = childIndex;
this.m_prunedChildren = new bool[ParetoMCTSParameters.NUM_ACTIONS];
this.m_numIters = 0;
//this.m_pi = pi;
isTerminal = false;
isExpanded = false;
isExhausted = false;
numExpandedChildren = 0;
numExhaustedChildren = 0;
nodeNum = GameManager.Instance.nodeCount;
GameManager.Instance.nodeCount++;
totValue = new double[ParetoMCTSParameters.NUM_TARGETS];
}
//public PlayoutInfo m_pi;
//public ParetoTreeNode()
//{
// this(null, null, -1, null, null, null, null,null);
//}
//public ParetoTreeNode(Game state, Roller roller, ParetoTreePolicy treePolicy, Random rnd,
// Player a_player, PlayoutInfo pi) {
// this(state, null, -1, roller, treePolicy, rnd, a_player,pi);
//}
//public ParetoTreeNode(Playfield state, RandomRoller roller, ParetoTreePolicy treePolicy, Random rnd,
// ParetoMCTSPlayer a_player)
//{
// this(state, null, -1, roller, treePolicy, rnd, a_player);
//}
//public ParetoTreeNode(Game state, ParetoTreeNode parent, int childIndex, Roller roller,
// TreePolicy treePolicy, Random rnd, Player a_player, PlayoutInfo pi) {
// this.m_player = a_player;
// this.state = state;
// this.parent = parent;
// this.m_rnd = rnd;
// children = new ParetoTreeNode[ParetoMCTSParameters.NUM_ACTIONS];
// totValue = new double[ParetoMCTSParameters.NUM_TARGETS];
// this.roller = roller;
// this.treePolicy = treePolicy;
// pa = new ParetoArchive();
// this.childIndex = childIndex;
// this.m_prunedChildren = new bool[ParetoMCTSParameters.NUM_ACTIONS];
// this.m_numIters = 0;
// this.m_pi = pi;
// if(parent == null) //This is only for the root:
// {
// this.initValueRoute();
// }
//}
public ParetoTreeNode(Playfield state, ParetoTreeNode parent, int childIndex, RandomRoller roller,
ParetoTreePolicy treePolicy, Random rnd, ParetoMCTSPlayer a_player)
{
this.m_player = a_player;
this.state = state;
this.parent = parent;
this.m_rnd = rnd;
//children = new ParetoTreeNode[ParetoMCTSParameters.NUM_ACTIONS];
this.roller = roller;
this.mTreePolicy = treePolicy;
pa = new ParetoArchive();
this.childIndex = childIndex;
this.m_prunedChildren = new bool[ParetoMCTSParameters.NUM_ACTIONS];
this.m_numIters = 0;
//this.m_pi = pi;
isTerminal = false;
isExpanded = false;
isExhausted = false;
numExpandedChildren = 0;
numExhaustedChildren = 0;
nodeNum = GameManager.Instance.nodeCount;
GameManager.Instance.nodeCount++;
totValue = new double[ParetoMCTSParameters.NUM_TARGETS];
//if (parent == null) //This is only for the root:
//{
// this.initValueRoute();
//}
}
public ParetoTreeNode treePolicy(ParetoTreeNode root)
{
ParetoTreeNode cur = root;
int depth = 0;
while (keepTreePolicy(cur, depth))
{
//ParetoTreeChanceNode chanceCur = cur as ParetoTreeChanceNode;
//if (chanceCur != null) {
// cur = chanceCur.doChanceAction(); //roll a dice
//}
//else
//{
cur = cur.bestChild();
depth++;
//}
m_runList.Insert(0, cur);
}
if (isExpandable(cur, depth))
{
cur.expand();
}
return(cur);
}
public ParetoTreeChanceNode(Playfield state, ParetoTreeNode parent, int childIndex, RandomRoller roller,
ParetoTreePolicy treePolicy, Random rnd, ParetoMCTSPlayer a_player, Action chanceAction, int childrenSize) :
this(state, parent, childIndex, roller,
treePolicy, rnd, a_player, chanceAction)
{
this.children = new List <ParetoTreeNode>(childrenSize);
}
//public override void expand()
//{
// bool lethalCheck = false;
// Playfield afterState = new Playfield(state);
// Movegenerator.Instance.getMoveListForPlayfield(afterState, false, lethalCheck);
// isExpanded = true;
// //List<Action> testMoves = Movegenerator.Instance.getMoveList(afterState, lethalCheck, true, true);
// if (afterState.moveList.Count > 0)
// {
// children = new ParetoTreeNode[afterState.moveList.Count];
// int i = 0;
// ParetoTreeNode tn = null;
// foreach (Action a in afterState.moveList)
// {
// //TODO: 暂时没有智慧祝福，收割机，大哥, 只有抽牌
// Playfield nextState = new Playfield(afterState);
// int cardDraw = nextState.getNumCardDraw();
// nextState.doAction(a);
// if (nextState.getNumCardDraw() != cardDraw) //it's a chance node
// {
// tn = new ParetoTreeChanceNode(new Playfield(nextState), this, i, this.roller, this.mTreePolicy, this.m_rnd, this.m_player, a);
// }
// tn = new ParetoTreeNode(nextState, this, i, this.roller, this.mTreePolicy, this.m_rnd, this.m_player);
// children[i] = tn;
// i++;
// }
// //Helpfunctions.Instance.logg("node expanded: " + this.nodeNum + ", children size: " + this.children.Length);
// //totValue = new double[afterState.moveList.Count];
// if (parent == null) //This is only for the root:
// {
// this.initValueRoute(afterState.moveList.Count);
// }
// }
// else
// {
// this.isTerminal = true;
// //this.isExhausted = true;
// //if (parent != null)
// //{
// // parent.numExhaustedChildren++;
// //}
// }
//}
public ParetoTreeNode doChanceAction() //only if it's a chance node
{
ParetoTreeNode tn;
Playfield nextState = new Playfield(this.parent.state);
nextState.doAction(chanceAction);
ActionResult ar = new ActionResult(nextState.moveTrigger.newHandcardList.ToArray());
if (this.drawCardResultMap == null) //init all the things
{
this.drawCardResultMap = new Dictionary <ActionResult, Tuple <double, int> >();
this.childrenCount = 1;
lastChildIndex = 0;
this.drawCardResultMap.Add(ar, new Tuple <double, int>(0, lastChildIndex));
tn = new ParetoTreeNode(nextState, this, lastChildIndex, this.roller, this.mTreePolicy, this.m_rnd, this.m_player);
this.children[lastChildIndex] = tn;
}
else if (!this.drawCardResultMap.ContainsKey(ar))
{
this.childrenCount++;
lastChildIndex = this.childrenCount - 1;
this.drawCardResultMap.Add(ar, new Tuple <double, int>(0, lastChildIndex));
tn = new ParetoTreeNode(nextState, this, lastChildIndex, this.roller, this.mTreePolicy, this.m_rnd, this.m_player);
this.children[lastChildIndex] = tn;
}
else
{
lastChildIndex = this.drawCardResultMap[ar].Item2;
tn = this.children[lastChildIndex];
}
return(tn);
}
public ParetoTreeNode bestChild(ParetoTreeNode node, double[][] bounds)
{
if (node.numExpandedChildren < node.children.Count)
{
List <int> childrenToSelect = new List <int>(node.children.Count);
int j = 0;
foreach (ParetoTreeNode child in node.children)
{
if (child.nVisits == 0)
{
childrenToSelect.Add(j);
}
j++;
}
node.numExpandedChildren++;
return(node.children[childrenToSelect[node.m_rnd.Next(childrenToSelect.Count)]]);
}
ParetoTreeNode selected = null;
double bestValue = -Double.MaxValue;
int i = 0;
foreach (ParetoTreeNode child in node.children)
{
//If it is not prunned.
if (!child.isExhausted)
{
double hvVal = child.getHV(false);
double childValue = hvVal;// / (child.nVisits + node.epsilon);
if (hvVal < 0)
{
Console.WriteLine("Negative HyperVolume: " + hvVal);
}
double uctValue = childValue +
K * Math.Sqrt(Math.Log(node.nVisits + 1) / (child.nVisits + node.epsilon)) +
node.m_rnd.NextDouble() * node.epsilon;
// small random numbers: break ties in unexpanded nodes
if (uctValue > bestValue)
{
selected = child;
bestValue = uctValue;
}
}
++i;
}
if (selected == null)
{
node.children[0].getHV(false);
throw new Exception("Warning! returning null: " + bestValue + " : " + node.children.Count);
}
return(selected);
}
public void mctsSearch(int a_timeDue, ParetoTreeNode root)
{ //TODO: prune bad boards: 1. we will die next turn, 2. we will die this turn
//long remaining = a_timeDue - DateTime.Now.Ticks / TimeSpan.TicksPerMillisecond;
int numIters = a_timeDue;
double invIters = 0.0;
for (int i = 0; i < numIters; i++)
{
m_runList.Clear();
m_runList.Add(root); //root always in.
if (root.isExhausted)
{
//Helpfunctions.Instance.logg("exit at num: " + i);
break; //if all exhausted, then return
}
ParetoTreeNode selected = treePolicy(root);
//double[] delta = selected.rollOut();
Playfield endTurnState = rollOut(selected);
//double[] delta = m_player.getHeuristic().value(endTurnState);
if (endTurnState.isOwnTurn == m_root.state.isOwnTurn)
{
int debug = 1;
}
//Debug.Assert(endTurnState.isOwnTurn != m_root.state.isOwnTurn);
double[] delta = getSolutionVector(endTurnState);
Solution deltaSol = new Solution(delta, endTurnState);
backUp(delta, deltaSol, true, selected.childIndex);
m_numIters++;
//remaining = a_timeDue - DateTime.Now.Ticks / TimeSpan.TicksPerMillisecond;
//if(treePolicy is ParetoEGreedyTreePolicy)
//{
// ((ParetoEGreedyTreePolicy) treePolicy).epsilon -= invIters;
//}
if (this.heuristicType == HeuristicType.LethalCheck && (deltaSol.m_data[0] == 1.0 || deltaSol.m_data[0] == 0)) //we got lethal
{
return;
}
}
//if (this.heuristicType == HeuristicType.DrawCard) //we got lethal
//{
// Helpfunctions.Instance.logg("drawcard size = " + root.pa.m_members.size());
// int debug = 1;
//}
//Helpfunctions.Instance.logg("normal exit: 500");
}
public Playfield rollOut(ParetoTreeNode tn)
{
switch (this.heuristicType)
{
case HeuristicType.Boardvalue:
return(tn.rollOut());
case HeuristicType.LethalCheck:
return(tn.rollOut());
case HeuristicType.DrawCard:
return(tn.chanceRollOut());
}
return(null);
}
public bool keepTreePolicy(ParetoTreeNode node, int depth)
{
switch (this.heuristicType)
{
case HeuristicType.Boardvalue:
return(!node.isLeaf() && !node.isExhausted && node.state.getGameResult() == -1 && depth < ParetoMCTSParameters.ROLLOUT_DEPTH);
case HeuristicType.LethalCheck:
return(!node.isLeaf() && !node.isExhausted && node.state.getGameResult() == -1 && depth < ParetoMCTSParameters.ROLLOUT_DEPTH);
case HeuristicType.DrawCard:
return(!node.isLeaf() && !node.isExhausted && node.state.getGameResult() == -1 &&
depth < ParetoMCTSParameters.ROLLOUT_DEPTH && (node.state.moveTrigger.newHandcardList.Count != 0 && depth > 0));
}
return(false);
}
public bool isExpandable(ParetoTreeNode pn, int depth)
{
switch (this.heuristicType)
{
case HeuristicType.Boardvalue:
return(!pn.isExpanded && pn.state.getGameResult() == -1);
case HeuristicType.LethalCheck:
return(!pn.isExpanded && pn.state.getGameResult() == -1);
case HeuristicType.DrawCard:
return(!pn.isExpanded && (depth == 0 || (depth > 0 && pn.state.moveTrigger.newHandcardList.Count == 0)));
//return !pn.isExpanded && pn.state.getGameResult() == -1 && pn.state.moveTrigger.newHandcardList.Count == 0;
}
return(true);
}
public void expand()
{
bool lethalCheck = false;
//if (m_player.heuristicType == HeuristicType.LethalCheck)
//lethalCheck = true;
isExpanded = true;
if (state.isOwnTurn != m_player.m_root.state.isOwnTurn)
{
return;
}
Movegenerator.Instance.getMoveListForPlayfield(state, false, lethalCheck, 0.0);
Playfield afterState = new Playfield(state);
if (afterState.moveList.Count > 0)
{
children = new List <ParetoTreeNode>(afterState.moveList.Count + 1);
int i = 0;
ParetoTreeNode tn = null;
//add endturn first
Playfield nextState = new Playfield(afterState);
//nextState.endTurn(false, false);
//tn = new ParetoTreeNode(nextState, this, i, this.roller, this.mTreePolicy, this.m_rnd, this.m_player);
//children.Add(tn);
foreach (Action a in afterState.moveList)
{
//TODO: 暂时没有智慧祝福，收割机，大哥, 只有抽牌
nextState = new Playfield(afterState);
nextState.doAction(a);
if (a.actionType == actionEnum.playcard && CardDB.Instance.UsefulNeedKeepDatabase.ContainsKey(a.card.card.name))
{
Playfield keepState = new Playfield(afterState);
keepState.keepCardList.Add(a.card.entity);
keepState.moveTrigger.keepCard = a.card.entity;
tn = new ParetoTreeNode(keepState, this, i, this.roller, this.mTreePolicy, this.m_rnd, this.m_player);
children.Add(tn);
i++;
}
int cardDrawThisTurn = nextState.moveTrigger.newHandcardList.Count;
tn = new ParetoTreeNode(nextState, this, i, this.roller, this.mTreePolicy, this.m_rnd, this.m_player);
children.Add(tn);
i++;
}
if (parent == null) //This is only for the root:
{
this.initValueRoute(i + 1);
}
}
else
{
this.isTerminal = true;
//this.isExhausted = true;
//if (parent != null)
//{
// parent.numExhaustedChildren++;
//}
}
}
//public ParetoMCTSPlayer(ParetoTreePolicy a_treePolicy, HeuristicMO a_h, Random a_rnd, Game a_game, PlayoutInfo pInfo)
//{
// m_playoutInfo = pInfo;
// //m_heightMap = new int[a_game.getMap().getMapWidth()][a_game.getMap().getMapHeight()];
// m_heuristic = a_h;
// m_heuristic.setPlayoutInfo(m_playoutInfo);
// m_treePolicy = a_treePolicy;
// this.m_rnd = a_rnd;
// this.m_targetWeights = ParetoMCTSParameters.targetWeights;
// m_globalPA = new ParetoArchive();
// m_randomRoller = new RandomRoller(RandomRoller.RANDOM_ROLLOUT, this.m_rnd, ParetoMCTSParameters.NUM_ACTIONS);
// m_root = new ParetoTreeNode(null, m_randomRoller, m_treePolicy, m_rnd, this, m_playoutInfo);
// this.m_numCalls = 0;
// this.m_numIters = 0;
//}
public ParetoMCTSPlayer(ParetoTreePolicy a_treePolicy, Random a_rnd, Playfield a_game, HeuristicType _ht)
{
//m_playoutInfo = pInfo;
////m_heightMap = new int[a_game.getMap().getMapWidth()][a_game.getMap().getMapHeight()];
//m_heuristic = a_h;
//m_heuristic.setPlayoutInfo(m_playoutInfo);
m_treePolicy = a_treePolicy;
this.m_rnd = a_rnd;
this.m_targetWeights = ParetoMCTSParameters.targetWeights;
m_globalPA = new ParetoArchive();
m_randomRoller = new RandomRoller(RandomRoller.RANDOM_ROLLOUT, this.m_rnd, ParetoMCTSParameters.NUM_ACTIONS);
//m_root = new ParetoTreeNode(null, m_randomRoller, m_treePolicy, m_rnd, this, m_playoutInfo);
m_root = new ParetoTreeNode(a_game, null, -1, m_randomRoller, m_treePolicy, m_rnd, this);
this.m_numCalls = 0;
this.m_numIters = 0;
m_runList = new List <ParetoTreeNode>();
heuristicType = _ht;
////bounds[0][0] = -Math.Sqrt(8) * 8 * 8;
//bounds[0][0] = 0;
//bounds[0][1] = Math.Sqrt(8) * 8 * 8 * 2;
////bounds[1][0] = -Math.Sqrt(8) * 8 * 8;
//bounds[1][0] = 0;
//bounds[1][1] = Math.Sqrt(8) * 8 * 8;
////bounds[2][0] = -10 * 10;
////bounds[2][1] = 10 * 10;
////bounds[2][0] = -Math.Sqrt(8) * 8 * 10;
//bounds[2][0] = 0;
//bounds[2][1] = Math.Sqrt(8) * 8 * 10 * 2;
switch (this.heuristicType)
{
case HeuristicType.Boardvalue:
bounds = new double[3][];
bounds[0] = new double[2];
bounds[1] = new double[2];
bounds[2] = new double[2];
//bounds[0][0] = -Math.Sqrt(8) * 8 * 8;
bounds[0][0] = 0;
bounds[0][1] = 1f;
//bounds[1][0] = -Math.Sqrt(8) * 8 * 8;
bounds[1][0] = 0;
bounds[1][1] = 1f;
//bounds[2][0] = -10 * 10;
//bounds[2][1] = 10 * 10;
//bounds[2][0] = -Math.Sqrt(8) * 8 * 10;
bounds[2][0] = 0;
bounds[2][1] = 1f;
break;
case HeuristicType.LethalCheck:
bounds = new double[1][];
bounds[0] = new double[2];
bounds[0][0] = 0;
bounds[0][1] = 1f;
break;
case HeuristicType.DrawCard:
bounds = new double[2][];
bounds[0] = new double[2];
bounds[1] = new double[2];
bounds[0][0] = 0;
bounds[0][1] = 1f;
bounds[1][0] = 0;
bounds[1][1] = 1f;
initNewCardCount = m_root.state.moveTrigger.newHandcardList.Count;
break;
}
}
public void backUp(double[] result, Solution sol, bool Added, int cI)
{
/*nVisits++;
* Added = pa.Add(result);
* int comingFrom = cI;
*
* for(int i = 0; i < result.Length; ++i)
* totValue[i] += result[i]; */
//for(ParetoTreeNode pn : m_runList)
int comingFrom = -1;
int numNodes = m_runList.Count;
for (int i = 0; i < numNodes; ++i)
{
ParetoTreeNode pn = m_runList[i];
//Helpfunctions.Instance.logg("node: " + pn.parent.nodeNum + ", children exhasuted: " + pn.parent.numExhaustedChildren + "/" + pn.parent.children.Length);
if (pn.isTerminal || (pn.children != null && pn.numExhaustedChildren == pn.children.Count))
{
pn.isExhausted = true;
if (pn.parent != null)
{
pn.parent.numExhaustedChildren++;
}
}
pn.nVisits++;
if (Added)
{
Added = pn.pa.add(sol);
//ParetoTreeChanceNode paretoChanceNode = pn.parent as ParetoTreeChanceNode;
//if (paretoChanceNode != null)
//{
// paretoChanceNode.HVvalue += pn.getHV(false);
//}
}
for (int j = 0; j < result.Length; ++j)
{
pn.totValue[j] += result[j];
}
if (i + 1 < numNodes)
{
//ParetoTreeNode parentNode = m_runList.get(i+1);
//parentNode.m_childCount[pn.childIndex]++; //for Nsa in one of the tree policies (see TransParetoTreePolicy).
comingFrom = pn.childIndex;
}
else if (i + 1 == numNodes)
{
if (pn.parent != null)
{
throw new Exception("This should be the root... and it's not.");
}
if (Added)
{
//Console.WriteLine("AddING (" + result[0] + "," + result[1] + ") to child " + comingFrom + " from " + pn.parent);
if (comingFrom != -1)
{
sol.m_through = comingFrom;
pn.valueRoute[comingFrom].Add(sol);
}
}
}
}
}
public void init()
{
//m_root = new ParetoTreeNode(null, m_randomRoller, m_treePolicy, m_rnd, this, m_playoutInfo);
m_root = new ParetoTreeNode(null, null, -1, m_randomRoller, m_treePolicy, m_rnd, this);
//m_heightMap = new int[m_heightMap.length][m_heightMap[0].length];
}
public void expand()
{
bool lethalCheck = false;
//if (m_player.heuristicType == HeuristicType.LethalCheck)
//lethalCheck = true;
Playfield afterState = new Playfield(state);
Movegenerator.Instance.getMoveListForPlayfield(afterState, false, lethalCheck);
//afterState.printMoveList();
//List<Action> testMoves = Movegenerator.Instance.getMoveList(afterState, lethalCheck, true, true);
//if (afterState.moveList.Count != testMoves.Count)
//{
// int debug = 1;
//}
isExpanded = true;
//List<Action> testMoves = Movegenerator.Instance.getMoveList(afterState, lethalCheck, true, true);
if (afterState.moveList.Count > 0)
{
children = new ParetoTreeNode[afterState.moveList.Count];
int i = 0;
foreach (Action a in afterState.moveList)
{
Playfield nextState = new Playfield(afterState);
nextState.doAction(a);
ParetoTreeNode tn = new ParetoTreeNode(nextState, this, i, this.roller, this.mTreePolicy, this.m_rnd, this.m_player);
children[i] = tn;
i++;
}
//Helpfunctions.Instance.logg("node expanded: " + this.nodeNum + ", children size: " + this.children.Length);
//totValue = new double[afterState.moveList.Count];
if (parent == null) //This is only for the root:
{
this.initValueRoute(afterState.moveList.Count);
}
}
else
{
this.isTerminal = true;
//this.isExhausted = true;
//if (parent != null)
//{
// parent.numExhaustedChildren++;
//}
}
}
