/// <summary>
/// Ballの生成
/// </summary>
private void CreateBall()
{
Pong.BallFactory factory = new Pong.BallFactory();
ball = Instantiate(factory.Create(), new Vector3(0, 0, 0), Quaternion.identity);
UnityEngine.Assertions.Assert.IsNotNull(ball);
ballScript = ball.GetComponent <Ball>();
UnityEngine.Assertions.Assert.IsNotNull(ballScript);
}
/// <summary>
/// ゴールイベント
/// </summary>
/// <param name="position">ゴールした方のプレイヤー</param>
public void Goal(PlayerConstant.Position position)
{
// Ball削除(オブザーバー解除してからオブジェクトを削除)
RequestRemoveObserver(ballScript);
Destroy(ball.gameObject);
ballScript = null;
ball = null;
if (position == PlayerConstant.Position.Left)
{
Debug.Log("GameTask(Goal) Left");
}
else
{
Debug.Log("GameTask(Goal) Right");
}
// スコアの加算
score.Add(position);
Debug.Log("score left " + score.Get(PlayerConstant.Position.Left));
Debug.Log("score right " + score.Get(PlayerConstant.Position.Right));
// スコア更新通知
scoreSubject.NotifyObservers(score);
// ゲーム終了したか
if (gameRule.CheckEnd(score))
{
// 文字列の設定
if (gameRule.GetWinner(score) == PlayerConstant.Position.Left)
{
SetResultLeftWin();
}
else
{
SetResultRightWin();
}
// タッチしたままEndにいった
if (touchAction.IsDragging() || touchAction.IsTouchStationary())
{
isResultTouchPushing = true;
}
scene = Scene.Ended;
}
else
{
scene = Scene.GoalSeStart;
}
}
public void tdUpdate(Ball b, string actionChosen)
{
// Q(s,a)
double currQVal = 0.0;
//var discreteStateTuple = Tuple.Create(b.DiscreteBallX, b.DiscreteBallY, b.DiscreteVelocityX, b.DiscreteVelocityY, this.DiscretePaddleY, actionChosen);
//double newPaddleY = this._paddleY;
//int discreteNewPaddleY = this._discretePaddleY;
//if (actionChosen.Equals("UP"))
//{
// newPaddleY -= 0.04;
// if (newPaddleY < 0)
// {
// newPaddleY = 0;
// }
// discreteNewPaddleY = (Int32)Math.Floor(this._boardY * newPaddleY / (1 - this._paddleHeight));
// var discreteStateTuple = Tuple.Create(b.DiscreteBallX, b.DiscreteBallY, b.DiscreteVelocityX, b.DiscreteVelocityY, discreteNewPaddleY);
//}
//else if (actionChosen.Equals("DOWN"))
//{
// newPaddleY = this._paddleY;
// if (newPaddleY > 1 - GlobalValues.PaddleHeight)
// {
// newPaddleY = 1 - GlobalValues.PaddleHeight;
// }
// discreteNewPaddleY = (Int32)Math.Floor(this._boardY * newPaddleY / (1 - this._paddleHeight));
// var discreteStateTuple = Tuple.Create(b.DiscreteBallX, b.DiscreteBallY, b.DiscreteVelocityX, b.DiscreteVelocityY, discreteNewPaddleY);
//}
// s
// Should I have moved it based on the action to get Q(s,a). Currently, no action was taken.
var discreteStateTuple = Tuple.Create(b.DiscreteBallX, b.DiscreteBallY, b.DiscreteVelocityX, b.DiscreteVelocityY, this.DiscretePaddleY);
try
{
this._Q.TryGetValue(discreteStateTuple, out currQVal);
}
catch
{
;
}
// R(s) Current Reward in initial state, regardless of action
int stateReward = 0;
if (b.BallX > this._paddleX)
{
// right player deflects it
// Calculate Y coordinate when ball is at the same level as the paddle
double yIntersect = b.VelocityY * b.BallX + b.BallY;
if (yIntersect <= this._paddleY + GlobalValues.PaddleHeight && yIntersect >= this._paddleY)
{
stateReward = 1;
}
else
{
stateReward = -1;
}
}
// Pretend to be in successor state
Ball nextBallState = new Pong.Ball(b.BallX, b.BallY, b.VelocityX, b.VelocityY, this._boardX, this._boardY, b.TwoPlayers);
nextBallState.MoveBall();
double newPaddleY = this._paddleY;
int discreteNewPaddleY = this.DiscretePaddleY;
if (actionChosen.Equals("UP"))
{
newPaddleY -= 0.04;
if (newPaddleY < 0)
{
newPaddleY = 0;
}
discreteNewPaddleY = (Int32)Math.Floor(this._boardY * newPaddleY / (1 - this._paddleHeight));
}
else if (actionChosen.Equals("DOWN"))
{
newPaddleY += 0.04;
if (newPaddleY > 1 - GlobalValues.PaddleHeight)
{
newPaddleY = 1 - GlobalValues.PaddleHeight;
}
discreteNewPaddleY = (Int32)Math.Floor(this._boardY * newPaddleY / (1 - this._paddleHeight));
}
else
{
// HOLD
// Do nothing
}
// s'
//var discreteSuccessorStateTuple = Tuple.Create(b.DiscreteBallX, b.DiscreteBallY, b.DiscreteVelocityX, b.DiscreteVelocityY, discreteNewPaddleY, actionChosen);
var discreteSuccessorStateTuple = Tuple.Create(nextBallState.DiscreteBallX, nextBallState.DiscreteBallY, nextBallState.DiscreteVelocityX, nextBallState.DiscreteVelocityY, discreteNewPaddleY, actionChosen);
// Get max Action in successor state
double upQ;
double downQ;
double holdQ;
moveDict.Clear();
// Simulate Next Decision, get Max Q
// Q(s',a')
//var discreteNextStateTupleUp = Tuple.Create(b.DiscreteBallX, b.DiscreteBallY, b.DiscreteVelocityX, b.DiscreteVelocityY, discreteNewPaddleY, "UP");
//this._Q.TryGetValue(discreteNextStateTupleUp, out upQ);
//moveDict.Add("UP", upQ);
//var discreteNextStateTupleDown = Tuple.Create(b.DiscreteBallX, b.DiscreteBallY, b.DiscreteVelocityX, b.DiscreteVelocityY, discreteNewPaddleY, "DOWN");
//this._Q.TryGetValue(discreteNextStateTupleDown, out downQ);
//moveDict.Add("DOWN", downQ);
//var discreteNextStateTupleHold = Tuple.Create(b.DiscreteBallX, b.DiscreteBallY, b.DiscreteVelocityX, b.DiscreteVelocityY, discreteNewPaddleY, "HOLD");
//this._Q.TryGetValue(discreteNextStateTupleHold, out holdQ);
//moveDict.Add("HOLD", holdQ);
// Get Successor State actions by changing Y
double nextPaddleY = newPaddleY;
int discreteNextPaddleY = discreteNewPaddleY;
// TODO: move the ball again?
// Get successor state UP action
nextPaddleY -= 0.04;
if (nextPaddleY < 0)
{
nextPaddleY = 0;
}
discreteNextPaddleY = (Int32)Math.Floor(this._boardY * nextPaddleY / (1 - this._paddleHeight));
var discreteNextStateTupleUp = Tuple.Create(nextBallState.DiscreteBallX, nextBallState.DiscreteBallY, nextBallState.DiscreteVelocityX, nextBallState.DiscreteVelocityY, discreteNextPaddleY);
this._Q.TryGetValue(discreteNextStateTupleUp, out upQ);
moveDict.Add("UP", upQ);
// Get successor state DOWN action
nextPaddleY = newPaddleY;
nextPaddleY += 0.04;
if (nextPaddleY > 1 - GlobalValues.PaddleHeight)
{
nextPaddleY = 1 - GlobalValues.PaddleHeight;
}
discreteNextPaddleY = (Int32)Math.Floor(this._boardY * nextPaddleY / (1 - this._paddleHeight));
var discreteNextStateTupleDown = Tuple.Create(nextBallState.DiscreteBallX, nextBallState.DiscreteBallY, nextBallState.DiscreteVelocityX, nextBallState.DiscreteVelocityY, discreteNextPaddleY);
this._Q.TryGetValue(discreteNextStateTupleDown, out downQ);
moveDict.Add("DOWN", upQ);
// Get successor state HOLD action
nextPaddleY = newPaddleY;
discreteNextPaddleY = (Int32)Math.Floor(this._boardY * nextPaddleY / (1 - this._paddleHeight));
var discreteNextStateTupleHold = Tuple.Create(nextBallState.DiscreteBallX, nextBallState.DiscreteBallY, nextBallState.DiscreteVelocityX, nextBallState.DiscreteVelocityY, discreteNextPaddleY);
this._Q.TryGetValue(discreteNextStateTupleHold, out downQ);
moveDict.Add("HOLD", upQ);
Dictionary <string, double> sortedMoveDict = moveDict.OrderByDescending(v => v.Value).ToDictionary(x => x.Key, x => x.Value);
// MaxQ(s',a')
double maxNextQValue = sortedMoveDict.First().Value;
var discreteStateActionTuple = Tuple.Create(discreteStateTuple.Item1, discreteStateTuple.Item2, discreteStateTuple.Item3, discreteStateTuple.Item4, discreteStateTuple.Item5, actionChosen);
// Number of times seen this state/action pair
int stateActionCount = 0;
try
{
this._N.TryGetValue(discreteStateActionTuple, out stateActionCount);
}
catch
{
;
}
double decayedLearningRate = this._learningConstant / (this._learningConstant + stateActionCount);
this._Q[discreteStateTuple] = currQVal + decayedLearningRate * (stateReward + _discountFactor * maxNextQValue - currQVal);
//// Check if states have value, else default it to 0
//double currQVal = 0.0;
//double nextQVal = 0.0;
//int learningRateCount = 0;
//// Calculate Q
//try
//{
// this._Q.TryGetValue(discreteStateTuple, out currQVal);
//}
//catch
//{
// ;
//}
//try
//{
// this._Q.TryGetValue(discreteSuccessorStateTuple, out nextQVal);
//}
//catch
//{
// ;
//}
//// Number of times seen this state/action pair
//try
//{
// this._N.TryGetValue(discreteStateTuple, out learningRateCount);
//}
//catch
//{
// ;
//}
//double decayedLearningRate = this._learningRate / (this._learningRate + learningRateCount);
//var tmpUp = Tuple.Create(6,6,1,0,6, "UP");
//var tmpDown = Tuple.Create(6, 6, 1, 0, 6, "DOWN");
//var tmpHold = Tuple.Create(6, 6, 1, 0, 6, "HOLD");
//nextQVal = maxNextQValue;
//this._Q[discreteStateTuple] = currQVal + decayedLearningRate * (stateReward + _discountFactor * nextQVal - currQVal);
}
/// <summary>
/// ゴール通知の解除リクエスト
/// </summary>
/// <param name="ballScript">Ballスクリプト</param>
private void RequestRemoveObserver(Pong.Ball ballScript)
{
ballScript.RequestRemoveObserver(this);
}
/// <summary>
/// ゴール通知の登録
/// </summary>
/// <param name="balScript">Ballスクリプト</param>
private void AddGoalObserver(Pong.Ball balScript)
{
ballScript.AddGoalObserver(this);
}