public void applyRewardForActionInState(double reward)
{
if (reward != 0 && priorState != null && priorAction != null && priorState.getActions() != null && priorState.getActions().Count() > 0)
{
List <Action> actions = priorState.getActions();
double priorQValue = priorAction.qValue;
FieldAction optimalAction = (FieldAction)actions[0];
foreach (Action action in actions)
{
if (action.qValue > optimalAction.qValue)
{
optimalAction = (FieldAction)action;
}
}
priorAction.qValue += learningRate * (reward + discountFactor * optimalAction.qValue - priorAction.qValue);
double growth = Math.Pow(Math.E, -Math.Abs(priorAction.qValue - priorQValue) / inverseSensitivity);
priorState.explorationRate = EXPLORATION_RATE_ACTION_WEIGHT * (1.0 - growth) / (1.0 + growth) + (1.0 - EXPLORATION_RATE_ACTION_WEIGHT) * priorState.explorationRate;
if (priorState.explorationRate > 1)
{
priorState.explorationRate = 1;
}
if (priorState.explorationRate < 0)
{
priorState.explorationRate = 0;
}
}
}
static string ToConstantFieldActionString(FieldAction value)
{
return(value switch
{
FieldAction.None => string.Empty,
FieldAction.Remove => "remove",
FieldAction.Keep => "keep",
_ => string.Empty
});
private List <FieldAction> generateActions()
{
List <FieldAction> actions = new List <FieldAction>();
for (int moveUp = 0; moveUp <= 1; ++moveUp)
{
for (int moveRight = 0; moveRight <= 1; ++moveRight)
{
for (int moveDown = 0; moveDown <= 1; ++moveDown)
{
for (int moveLeft = 0; moveLeft <= 1; ++moveLeft)
{
for (int rotateClockwise = 0; rotateClockwise <= 1; ++rotateClockwise)
{
for (int rotateCounterClockwise = 0; rotateCounterClockwise <= 1; ++rotateCounterClockwise)
{
for (int lookUp = 0; lookUp <= 1; ++lookUp)
{
for (int lookDown = 0; lookDown <= 1; ++lookDown)
{
for (int chargeAttachment = 0; chargeAttachment <= 1; ++chargeAttachment)
{
for (int useAttachment = 0; useAttachment <= 1; ++useAttachment)
{
for (int pickAttachment = 0; pickAttachment < NUMBER_OF_NON_PASSIVE_ATTACHMENTS; ++pickAttachment)
{
FieldAction action = new FieldAction();
action.moveUp = moveUp == 1;
action.moveRight = moveRight == 1;
action.moveDown = moveDown == 1;
action.moveLeft = moveLeft == 1;
action.rotateClockwise = rotateClockwise == 1;
action.rotateCounterClockwise = rotateCounterClockwise == 1;
action.lookUp = lookUp == 1;
action.lookDown = lookDown == 1;
action.chargeAttachment = chargeAttachment == 1;
action.useAttachment = useAttachment == 1;
action.pickAttachment = BOT.getAttachments()[pickAttachment];
actions.Add(action);
}
}
}
}
}
}
}
}
}
}
}
return(actions);
}
public FieldAction getActionForState(FieldState state)
{
if (priorTime == 0)
{
priorTime = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
}
long currentTime = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
priorTime = currentTime;
if (state != null)
{
priorState = state;
}
else
{
state = priorState;
}
if ((FIELD_STATES.Count() == 0) || !FIELD_STATES.Exists(s => s.Compare(state) == 0))
{
List <Action> actionsForState = null;
if (FIELD_STATES.Count == 0 || RANDOM.NextDouble() < priorState.explorationRate)
{
actionsForState = generateActionsForState();
}
else
{
actionsForState = getClosestState(state).getActions();
}
addStateActionPair(state, actionsForState);
}
else
{
priorState = FIELD_STATES.Find(s => s.Compare(state) == 0);
}
List <Action> actions = default;
if (FIELD_STATES.Count > 0)
{
actions = priorState.getActions();
}
if ((actions == default) || (FIELD_STATES.Count == 0) || ((priorState.explorationRate > 0) && (RANDOM.NextDouble() < priorState.explorationRate)))
{
int actionIndex = RANDOM.Next(MAX_ACTIONS);
if (actions != default && FIELD_STATES.Count > 0 && actionIndex < actions.Count())
{
priorAction = (FieldAction)actions[actionIndex];
}
}
public override bool equals(Action action)
{
FieldAction fieldAction = (FieldAction)action;
return(moveUp == fieldAction.moveUp &&
moveUp == fieldAction.moveUp &&
moveRight == fieldAction.moveRight &&
moveDown == fieldAction.moveDown &&
moveLeft == fieldAction.moveLeft &&
rotateClockwise == fieldAction.rotateClockwise &&
rotateCounterClockwise == fieldAction.rotateCounterClockwise &&
lookUp == fieldAction.lookUp &&
lookDown == fieldAction.lookDown &&
chargeAttachment == fieldAction.chargeAttachment &&
useAttachment == fieldAction.useAttachment &&
pickAttachment == fieldAction.pickAttachment);
}
public override void Play(int row, int column, FieldAction action)
{
var state = GameState;
base.Play(row, column, action);
if (state != GameState.Won && GameState == GameState.Won)
{
var official = _officialGamemodes.FirstOrDefault(g => g.Bombs == Gamemode.Bombs &&
g.Width == Gamemode.Width &&
g.Height == Gamemode.Height &&
g.Name == Gamemode.Name);
if (official != null)
{
_database.Add(Gamemode, _nickname, RoundTime, StopTime);
}
}
}
public AIAgent(Robot robot, List <BuildHubState> buildHubStates, List <FieldState> fieldStates, double discountFactor, double learningRate, double inverseSensitivity)
{
BOT = robot;
BUILD_HUB_STATES = buildHubStates;
FIELD_STATES = fieldStates;
if (FIELD_STATES == default)
{
FIELD_STATES = new List <FieldState>();
}
this.discountFactor = discountFactor;
this.learningRate = learningRate;
this.inverseSensitivity = inverseSensitivity;
priorState = null;
priorAction = null;
RANDOM = new Random();
NUMBER_OF_NON_PASSIVE_ATTACHMENTS = BOT.getAttachments().FindAll(attachment => !attachment.isPassive()).Count();
MAX_ACTIONS = (int)Math.Round(Math.Pow(2, typeof(FieldAction).GetFields().Length - 2)) * (NUMBER_OF_NON_PASSIVE_ATTACHMENTS + 1);
EXPLORATION_RATE_ACTION_WEIGHT = 1.0 / MAX_ACTIONS;
BASE_ACTIONS = generateActions();
}
private List <Action> actionsDataToActions(ActionData[] actions, List <Attachment> parts)
{
List <Action> actionList = new List <Action>();
foreach (FieldActionData actionData in actions)
{
FieldAction action = new FieldAction();
action.moveUp = actionData.moveUp;
action.moveRight = actionData.moveRight;
action.moveDown = actionData.moveDown;
action.moveLeft = actionData.moveLeft;
action.rotateClockwise = actionData.rotateClockwise;
action.rotateCounterClockwise = actionData.rotateCounterClockwise;
action.lookUp = actionData.lookUp;
action.lookDown = actionData.lookDown;
action.chargeAttachment = actionData.chargeAttachment;
action.useAttachment = actionData.useAttachment;
action.pickAttachment = parts.Find(p => p.getID() == actionData.pickAttachment.id);
action.qValue = actionData.qValue;
actionList.Add(action);
}
return(actionList);
}
public virtual void Play(int row, int column, FieldAction action)
{
if (GameState == GameState.Lost || GameState == GameState.Won ||
row < 0 || row >= Gamemode.Height ||
column < 0 || column >= Gamemode.Width)
{
return;
}
if (GameState == GameState.New && action == FieldAction.Open)
{
Setup(row, column);
}
switch (action)
{
case FieldAction.Open:
Open(row, column);
CheckGamestate(row, column);
break;
case FieldAction.Flag:
Flag(row, column);
break;
case FieldAction.Mark:
Mark(row, column);
break;
case FieldAction.Reset:
Reset(row, column);
break;
default:
throw new ArgumentException();
}
}
public FormFieldActionAttribute(string Title, string Command)
{
this.FieldAction = new FieldAction(Title, Command);
}
private FieldState initializeState(FieldState priorState, FieldAction priorAction, Robot bot, Robot[] enemies, int agentIndex)
{
FieldState state = new FieldState();
state.enemiesKilled = new bool[NUMBER_OF_ENEMIES];
state.enemiesPosition = new Point[NUMBER_OF_ENEMIES];
state.enemiesCanHit = new bool[NUMBER_OF_ENEMIES];
int numberOfAttachments = 0;
Camera camera = ROBOT_CAMERAS[agentIndex];
if (camera == default)
{
camera = findCameraForRobot(bot);
ROBOT_CAMERAS[agentIndex] = camera;
}
Plane[] planes = GeometryUtility.CalculateFrustumPlanes(camera);
Collider collider = null;
AGENT_POSITIONS[agentIndex] = new Point(bot.GAME_OBJECT.transform.position);
if (priorState == null)
{
state.killed = bot.getRemainingDurability() <= 0;
state.enemiesTouchingObstacles = new bool[NUMBER_OF_ENEMIES][];
if (enemies.Length > 0)
{
for (int enemyIndex = 0; enemyIndex < NUMBER_OF_ENEMIES; ++enemyIndex)
{
state.enemiesKilled[enemyIndex] = enemies[enemyIndex].getRemainingDurability() <= 0;
AGENT_ENEMY_POSITIONS[agentIndex][enemyIndex] = new Point(enemies[enemyIndex].GAME_OBJECT.transform.position);
state.enemiesPosition[enemyIndex] = generateLocationRelativeToCrosshair(bot, enemies[enemyIndex], camera, AGENT_ENEMY_POSITIONS[agentIndex][enemyIndex]);
state.enemiesTouchingObstacles[enemyIndex] = new bool[NUMBER_OF_OBSTACLES];
}
}
state.timeSinceLastAction = 0;
Attachment[] attachments = bot.getAttachments().FindAll(attachment => !attachment.isPassive()).ToArray();
state.attachments = attachments;
state.enemiesLowDurability = new bool[NUMBER_OF_ENEMIES];
state.canSeeEnemies = new bool[NUMBER_OF_ENEMIES];
state.touchingObstacles = new bool[NUMBER_OF_OBSTACLES];
state.numberOfEnemies = NUMBER_OF_ENEMIES;
numberOfAttachments = attachments.Length;
state.attachmentsCharging = new bool[numberOfAttachments];
state.attachmentsCooling = new bool[numberOfAttachments];
}
else
{
state.damageReceived = (PRIOR_DURABILITY_REMAINING[agentIndex] - bot.getRemainingDurability()) > 0 ? true : false;
PRIOR_DURABILITY_REMAINING[agentIndex] = bot.getRemainingDurability();
state.damageDealt = (bot.getDamageDealt() - PRIOR_DAMAGE_DEALT[agentIndex]) != 0 ? true : false;
PRIOR_DAMAGE_DEALT[agentIndex] = bot.getDamageDealt();
state.killed = (bot.getRemainingDurability() <= 0) != priorState.killed;
state.enemiesTouchingObstacles = priorState.enemiesTouchingObstacles;
if (enemies.Length > 0)
{
for (int enemyIndex = 0; enemyIndex < NUMBER_OF_ENEMIES; ++enemyIndex)
{
state.enemiesKilled[enemyIndex] = (enemies[enemyIndex].getRemainingDurability() <= 0) != priorState.enemiesKilled[enemyIndex];
AGENT_ENEMY_POSITIONS[agentIndex][enemyIndex] = new Point(enemies[enemyIndex].GAME_OBJECT.transform.position);
state.enemiesPosition[enemyIndex] = generateLocationRelativeToCrosshair(bot, enemies[enemyIndex], camera, AGENT_ENEMY_POSITIONS[agentIndex][enemyIndex]);
}
}
if (priorAction == null)
{
state.timeSinceLastAction = 0;
}
else if (priorAction.pickAttachment == null &&
!priorAction.moveUp &&
!priorAction.moveRight &&
!priorAction.moveDown &&
!priorAction.moveLeft &&
!priorAction.rotateClockwise &&
!priorAction.rotateCounterClockwise &&
!priorAction.lookUp &&
!priorAction.lookDown &&
!priorAction.chargeAttachment &&
!priorAction.useAttachment)
{
++state.timeSinceLastAction;
}
else
{
state.timeSinceLastAction = 0;
}
state.attachments = priorState.attachments;
state.enemiesLowDurability = priorState.enemiesLowDurability;
state.canSeeEnemies = priorState.canSeeEnemies;
state.touchingObstacles = priorState.touchingObstacles;
state.numberOfEnemies = NUMBER_OF_ENEMIES;
numberOfAttachments = state.attachments.Length;
state.attachmentsCharging = priorState.attachmentsCharging;
state.attachmentsCooling = priorState.attachmentsCooling;
}
state.lowDurability = bot.getRemainingDurability() / bot.getDurability() <= .3f;
Point botPosition = new Point(bot.GAME_OBJECT.transform.position);
Dimension botSize = bot.getSize();
state.touchingWall = botPosition.x >= (FIELD_SIZE.width / 2 - botSize.width) || botPosition.x <= (-FIELD_SIZE.width / 2 + botSize.width) ||
botPosition.z >= (FIELD_SIZE.depth / 2 - botSize.depth) || botPosition.z <= (-FIELD_SIZE.depth / 2 + botSize.width);
if (enemies.Length > 0)
{
for (int enemyIndex = 0; enemyIndex < NUMBER_OF_ENEMIES; ++enemyIndex)
{
state.enemiesLowDurability[enemyIndex] = enemies[enemyIndex].getRemainingDurability() <= .3f;
state.canSeeEnemies[enemyIndex] = enemies[enemyIndex].isVisible();
if (state.canSeeEnemies[enemyIndex])
{
float distanceToRobot = 0f;
float hitDistance = default;
distanceToRobot = Vector3.Distance(camera.transform.position, AGENT_ENEMY_POSITIONS[agentIndex][enemyIndex].toVector3());
hitDistance = getDistanceToObstruction(camera);
collider = enemies[enemyIndex].GAME_OBJECT.GetComponent <Collider>();
state.canSeeEnemies[enemyIndex] = hitDistance != Mathf.Infinity && enemies[enemyIndex].isVisible() && GeometryUtility.TestPlanesAABB(planes, collider.bounds) && distanceToRobot == hitDistance;
}
state.enemiesCanHit[enemyIndex] = canHit(enemies[enemyIndex], bot, camera, AGENT_POSITIONS[agentIndex]);
for (int obstacleIndex = 0; obstacleIndex < NUMBER_OF_OBSTACLES; ++obstacleIndex)
{
List <Collider> collidersTouchingObstacle = OBSTACLES[obstacleIndex].getCollidersTouching();
Collider robotCollider = enemies[enemyIndex].GAME_OBJECT.GetComponent <Collider>();
Collider robotHeadCollider = enemies[enemyIndex].getHead().GAME_OBJECT.GetComponent <Collider>();
Collider robotBodyCollider = enemies[enemyIndex].getBody().GAME_OBJECT.GetComponent <Collider>();
Collider robotMobilityCollider = enemies[enemyIndex].getMobility().GAME_OBJECT.GetComponent <Collider>();
state.enemiesTouchingObstacles[enemyIndex][obstacleIndex] = collidersTouchingObstacle.Contains(robotCollider) || collidersTouchingObstacle.Contains(robotHeadCollider) || collidersTouchingObstacle.Contains(robotBodyCollider) || collidersTouchingObstacle.Contains(robotMobilityCollider);
}
}
}
for (int obstacleIndex = 0; obstacleIndex < NUMBER_OF_OBSTACLES; ++obstacleIndex)
{
List <Collider> collidersTouchingObstacle = OBSTACLES[obstacleIndex].getCollidersTouching();
Collider robotCollider = bot.GAME_OBJECT.GetComponent <Collider>();
Collider robotHeadCollider = bot.getHead().GAME_OBJECT.GetComponent <Collider>();
Collider robotBodyCollider = bot.getBody().GAME_OBJECT.GetComponent <Collider>();
Collider robotMobilityCollider = bot.getMobility().GAME_OBJECT.GetComponent <Collider>();
state.touchingObstacles[obstacleIndex] = collidersTouchingObstacle.Contains(robotCollider) || collidersTouchingObstacle.Contains(robotHeadCollider) || collidersTouchingObstacle.Contains(robotBodyCollider) || collidersTouchingObstacle.Contains(robotMobilityCollider);
}
Attachment[] attachmentsInState = state.attachments;
numberOfAttachments = attachmentsInState.Length;
for (int attachmentIndex = 0; attachmentIndex < numberOfAttachments; ++attachmentIndex)
{
state.attachmentsCharging[attachmentIndex] = attachmentsInState[attachmentIndex].getElapsedChargeTime() < attachmentsInState[attachmentIndex].getMaxChargeTime();
state.attachmentsCooling[attachmentIndex] = !attachmentsInState[attachmentIndex].isCooled();
}
return(state);
}
