static void Main(string[] args)
{
QAgent <int, int> agent = new QAgent <int, int>();
int secretNumber = 5;
//for each state we can add 1 and subtract 1
agent.RequestActionsForState = (int state) => new List <int>()
{
-1, 1
};
for (int i = 0; i < 10; i++)
{
int guessedNumber = 0;
Console.WriteLine("--------------------------");
do
{
//flag if game is finished
bool finish = false;
int prevstate = guessedNumber;
var action = agent.GetAction(guessedNumber);
var reward = -0.1;
//execute action
guessedNumber += action;
Console.WriteLine(guessedNumber);
//specify the rule that guessed number cant be less than 0
if (guessedNumber < 0)
{
reward = -1;
finish = true;
}
//rule for victory
if (guessedNumber == secretNumber)
{
reward = 1;
finish = true;
}
//learn what would happen after action is executed
agent.UpdateState(prevstate, action, reward, guessedNumber);
//finish the game
if (finish)
{
break;
}
} while (true);
}
Console.ReadLine();
}
public TTTModule() : base("/ttt")
{
QAgent <string, int> agentX = new QAgent <string, int>();
agentX.RequestActionsForState = SpaceIndexes;
QAgent <string, int> agentO = new QAgent <string, int>();
agentO.RequestActionsForState = SpaceIndexes;
Post("/predict", o =>
{
var name = this.Request.Query["name"].ToString();
var reader = new StreamReader(Request.Body);
var state = reader.ReadToEnd();
if (File.Exists($"agent{name}.json"))
{
agentX.Deserialize(File.ReadAllText($"agent{name}.json"));
}
return(agentX.GetAction(state).ToString());
});
Post("/learn", o =>
{
var name = this.Request.Query["name"].ToString();
if (File.Exists($"agent{name}.json"))
{
agentX.Deserialize(File.ReadAllText($"agent{name}.json"));
}
agentX.UpdateState(this.Request.Form["prevState"].ToString(), int.Parse(this.Request.Form["action"].ToString()), int.Parse(this.Request.Form["reward"].ToString()), this.Request.Form["newState"].ToString());
File.WriteAllText($"agent{name}.json", agentX.Serialize());
return("OK");
});
}
public LabyrinthModule() : base("/labyrinth")
{
QAgent <int, string> agent = new QAgent <int, string>();
agent.RequestActionsForState = i =>
{
var res = new List <string>()
{
"up", "down", "left", "right"
};
int y = i / 10;
int x = i - y * 10;
if (x == 0)
{
res.Remove("left");
}
if (x == 4)
{
res.Remove("right");
}
if (y == 4)
{
res.Remove("down");
}
if (y == 0)
{
res.Remove("up");
}
return(res);
};
Post("/predict", o =>
{
var name = this.Request.Query["name"].ToString();
var reader = new StreamReader(Request.Body);
var state = reader.ReadToEnd();
if (File.Exists($"agent{name}.json"))
{
agent.Deserialize(File.ReadAllText($"agent{name}.json"));
}
return(agent.GetAction(int.Parse(state)).ToString());
});
Post("/learn", o =>
{
var name = this.Request.Query["name"].ToString();
if (File.Exists($"agent{name}.json"))
{
agent.Deserialize(File.ReadAllText($"agent{name}.json"));
}
agent.UpdateState(int.Parse(this.Request.Form["prevState"].ToString()), this.Request.Form["action"].ToString(), int.Parse(this.Request.Form["reward"].ToString()), int.Parse(this.Request.Form["newState"].ToString()));
File.WriteAllText($"agent{name}.json", agent.Serialize());
return("OK");
});
Post("/overlay", o =>
{
var name = this.Request.Query["name"].ToString();
if (File.Exists($"agent{name}.json"))
{
agent.Deserialize(File.ReadAllText($"agent{name}.json"));
}
double[][] res = new double[5][];
for (int y = 0; y < 5; y++)
{
res[y] = new double[5];
for (int x = 0; x < 5; x++)
{
var state = y * 10 + x;
res[y][x] = agent.GetStateMaxScore(state);
}
}
return(JsonConvert.SerializeObject(res));
});
}
public virtual void Update()
{
//if (healthController.health > 0) {
UpdateMovement();
UpdateOrientation();
UpdateFriendlyFireFlag();
timeSinceLastUpdatingCurrentPath += Time.deltaTime;
timeSinceLastUpdate += Time.deltaTime;
timeSinceLastActionUpdate += Time.deltaTime;
// if (GameData.scores [0] != expectedScores [0] || GameData.scores [1] != expectedScores [1]) {
// CurrentTarget = null;
// expectedScores [0] = GameData.scores [0];
// expectedScores [1] = GameData.scores [1];
// }
/*
* This code runs unless we have set up the system to give reward based on the reward the player gives
*/
if (autoSetReward)
{
reward += healthRewardMultiplier * (healthController.health / healthController.maxhealth) * (Time.deltaTime);
if (Vector3.SqrMagnitude(gameObject.transform.position - previousPos) > posRewardThreshold * posRewardThreshold)
{
previousPos = gameObject.transform.position;
reward += (posRewardThreshold / agent.speed) * hasPathReward;
}
FactionName[] activeFactions = GameData.ActiveFactions();
for (int i = 0; i < activeFactions.Length; ++i)
{
if (activeFactions [i] == thisEntity.faction)
{
continue;
}
//if (scoreDiff<0){
reward += ((GameData.scores[thisEntity.faction] - GameData.scores[activeFactions [i]]) * Time.deltaTime);
}
//}
if (autoSetReward && currentTargetIsVisible == LOSResult.Visible && weapon.stateOfWeapon == WeaponState.Reloading)
{
reward -= (Time.deltaTime * 10);
}
if (currentTarget != null && currentTarget.mainLOSCollider != null)
{
// if (previousTargetHealth < currentTarget.controlHealth.health) {
// qAgent.RewardAgent (10);
// previousTargetHealth = currentTarget.controlHealth.maxhealth;
// }
// if (currentTarget.controlHealth.health < previousTargetHealth) {
// reward += (previousTargetHealth - currentTarget.controlHealth.health);
// previousTargetHealth = currentTarget.controlHealth.health;
//
// }
}
}
/*
* This code makes the agent more efficient by making it not update on every frame
*/
if (timeSinceLastUpdate > AIUpdateInterval)
{
timeSinceLastUpdate %= AIUpdateInterval;
UpdateLOSInfo();
}
/*
* Get a new action, if an appropriate interval has passed after getting the last action
*/
if (timeSinceLastActionUpdate > actionUpdateInterval)
{
//if (!(weapon.stateOfWeapon == WeaponState.Reloading)) {
timeSinceLastActionUpdate %= actionUpdateInterval; // this statement gets moved here so that only if the next action is requested do we restart the timer before requesting an action
if (autoSetReward)
{
qAgent.RewardAgent(reward); //we call the q agent reward directly, as we don't want to pointlessly add another function call to the stack by calling the reward method of this class
reward = 0;
}
currentActionSet = qAgent.GetAction(getState(), getStateValues());
/*
* We immediately apply the action of turning 180 degrees
*/
if (currentActionSet.Contains("Turn180"))
{
Vector3 c_dest = agent.destination;
gameObject.transform.Rotate(new Vector3(0, 180, 0));
/*
* we need to put the agent back on the navmesh in case rotating
* it put the agent off the navmesh. This can happen if the navmesh
* modek has a different centre to the agent
*/
agent.SetDestination(c_dest);
}
//}
}
//thisEntity.faction
//if (shuffleIndex % 2 == 0) {
if (currentActionSet.Contains("Shoot") && weapon.stateOfWeapon == WeaponState.Ready && (!useCheatConditions || (weapon.magazines.Count > 0 && weapon.magazines [weapon.currentMag] > 0)))
{
/*
* Punish agent if it tries to shoot while having no ammo. This is meant to encourage it to reload
*/
if (autoSetReward && (weapon.magazines.Count == 0 || weapon.magazines [weapon.currentMag] == 0))
{
reward -= Time.deltaTime;
}
else if ((currentTarget != null && currentTarget.mainLOSCollider != null))
{
reward += Time.deltaTime;
}
/*
* If a enemy is in sight we point bullets towards the enemy even
* if the barrel doesn't face the enemy, assuming that the angle
* that the bullet is adjusted by
* (the angle between the barrel and the line from this agent to the enemy)
* isn't too large.
* If the angle is too large or if there is no opponent, the bullet follows the
* direction of the gun barrel
*/
if (CurrentTarget == null || Vector3.Angle(gameObject.transform.forward, (currentTarget.mainLOSCollider.gameObject.transform.position - weapon.barrelStart.position)) > maxAngleOffsetForFiringShotsNotLinedUpWithBarrel)
{
weapon.fire(weapon.barrelEnd.position, weapon.barrelStart.position, thisEntity.faction, false);
}
else
{
weapon.fire(((currentTarget.mainLOSCollider.gameObject.transform.position - weapon.barrelStart.position).normalized * (weapon.barrelEnd.position - weapon.barrelStart.position).magnitude) + weapon.barrelStart.position, weapon.barrelStart.position, thisEntity.faction, false);
}
}
else
/*
* If out of ammo, reload
*/
if (currentActionSet.Contains("Reload") && weapon.stateOfWeapon != WeaponState.Reloading && (!useCheatConditions || (weapon.magazines.Count == 0 || weapon.magazines [weapon.currentMag] < weapon.magSize * 0.5f)))
{
StartCoroutine(Reload());
}
// } else {
// if (currentActionSet.Contains ("SHOOT") && weapon.stateOfWeapon == WeaponState.Ready) {
// weapon.fire(weapon.barrelEnd.position,weapon.barrelStart.position,thisEntity.faction,false);
// } else if (currentActionSet.Contains ("RELOAD") && weapon.stateOfWeapon!=WeaponState.Reloading) {
// StartCoroutine(Reload());
// }
// }
/* apply action for healing */
if (currentActionSet.Contains("BigHeal"))
{
heal(false);
}
else if (currentActionSet.Contains("Heal"))
{
heal(true);
}
}