public override Move Move(Virus percept) {
//Stopwatch watch = new Stopwatch();
//watch.Start();
VirusBoard currentState = percept.GetBoardCopy();
Move[] actions = currentState.GetPossibleMoves(playerNumber);
Move action = actions[0];
double max = double.NegativeInfinity;
foreach (Move a in actions)
{
VirusBoard newState = currentState.GetUpdated(a);
double q = Utility(currentState, newState);
q += MinValue(newState, 0);
if (q > max)
{
max = q;
action = a;
}
if (max == double.PositiveInfinity) {
break;
}
}
//watch.Stop();
//StreamWriter timeWriter = new StreamWriter("mmTimeLog",true);
//timeWriter.WriteLine(watch.ElapsedMilliseconds); // + " ; " + watch.ElapsedTicks);
//timeWriter.Close();
return action;
}
static void Main(String[] args) {
int boardsize = 4;
int tilesize = 40;
bool immediateness = true;
String[] players = new String[] { "AnnAI", "Player" };
if (args.Length > 0) {
boardsize = int.Parse(args[0]);
}
if (args.Length > 1) {
tilesize = int.Parse(args[1]);
}
if (args.Length > 2) {
immediateness = bool.Parse(args[2]);
}
if (args.Length > 3) {
players = new String[args.Length - 3];
for (int i = 0; i < players.Length; i++) {
players[i] = args[i + 3];
}
}
Application.EnableVisualStyles();
Application.SetCompatibleTextRenderingDefault(false);
Virus virus = new Virus(players.Length, boardsize);
Application.Run(new VirusInterface(virus, tilesize, immediateness, players));
}
public override void EndGame(Virus percept) {
if (learn) {
double reward = 0;
byte winner = percept.Winner;
if (winner == playerNumber)
reward = 1;
else if (winner != playerNumber && winner != 0)
reward = -1;
else
reward = 0;
if (!N.ContainsKey(prevState.CustomHash()))
N.Add(prevState.CustomHash(), new Dictionary<UInt32, int>());
if (!N[prevState.CustomHash()].ContainsKey(prevAction.CustomHash()))
N[prevState.CustomHash()].Add(prevAction.CustomHash(), 0);
N[prevState.CustomHash()][prevAction.CustomHash()]++;
Q[prevState.CustomHash()][prevAction.CustomHash()] =
Q[prevState.CustomHash()][prevAction.CustomHash()]
+ LearningRate(N[prevState.CustomHash()][prevAction.CustomHash()])
* (reward - Q[prevState.CustomHash()][prevAction.CustomHash()]);
}
prevState = default(VirusBoard);
prevAction = default(Move);
prevReward = 0;
}
public override Move Move(Virus percept) {
VirusBoard currentState = percept.GetBoardCopy();
Move[] actions = currentState.GetPossibleMoves(playerNumber);
Move action = actions[0];
double max = double.NegativeInfinity;
foreach (Move a in actions) {
VirusBoard newState = currentState.GetUpdated(a);
double q = 0;
if (Q.ContainsKey(currentState.CustomHash())) {
if (Q[currentState.CustomHash()].ContainsKey(a.CustomHash())) {
q = Q[currentState.CustomHash()][a.CustomHash()];
}
}
q += MinValue(newState, 0);
if (q > max) {
max = q;
action = a;
}
if (max == 1) {
break;
}
}
return action;
}
static void comTestMemQ() {
//train a MemQ agent, and once every ten games have it teach an observer
//do this with ten agents
const int teachingagents = 10;
const int gamesbetweenteaching = 10;
const int learninggames = 10000;
int wins = 0;
int n = 0;
while (File.Exists("observer" + n + ".log")) n++;
StreamWriter writer = new StreamWriter("observer" + n + ".log");
writer.WriteLine("sources");
Virus virus;
MemoryQAgent observer = new MemoryQAgent(1);
Agent opponent = new BruteForceAgent(2);
for (int i = 0; i < teachingagents; i++) {
MemoryQAgent agent = new MemoryQAgent(1);
wins = 0;
for (int j = 0; j < learninggames; j++) {
virus = new Virus(2, 5);
wins += RunGame(virus, agent, opponent) == 1 ? 1 : 0;
agent.ProcessShortTermMemory();
if (j % gamesbetweenteaching == gamesbetweenteaching - 1) {
agent.TellOfMemoryTo(observer, true);
}
if (j % 1000 == 999) {
if (j > 9500) {
writer.WriteLine(wins);
Console.WriteLine("Wins: " + wins);
}
wins = 0;
}
}
}
writer.WriteLine("observer");
for (int i = 0; i < 10; i++) {
wins = 0;
for (int j = 0; j < 1000; j++) {
virus = new Virus();
wins += RunGame(virus, observer, opponent) == 1 ? 1 : 0;
observer.ForgetShortTerm();
}
writer.WriteLine(wins);
Console.WriteLine("Observer wins: " + wins);
}
writer.Close();
}
public override Move Move(Virus percept) {
//Checking if we're at an terminal state
byte winner = percept.Winner;
VirusBoard newState = percept.GetBoardCopy();
if (!Q.ContainsKey(newState.CustomHash()))
Q.Add(newState.CustomHash(), new Dictionary<UInt32, double>());
if (learn && !prevState.Equals(default(VirusBoard))) {
if (!N.ContainsKey(prevState.CustomHash()))
N.Add(prevState.CustomHash(), new Dictionary<UInt32, int>());
if (!N[prevState.CustomHash()].ContainsKey(prevAction.CustomHash()))
N[prevState.CustomHash()].Add(prevAction.CustomHash(), 0);
if (!Q.ContainsKey(prevState.CustomHash()))
Q.Add(prevState.CustomHash(), new Dictionary<UInt32, double>());
if (!Q[prevState.CustomHash()].ContainsKey(prevAction.CustomHash()))
Q[prevState.CustomHash()].Add(prevAction.CustomHash(), initvalue);
if (winner == playerNumber) {
if (!Q[newState.CustomHash()].ContainsKey(0))
Q[newState.CustomHash()].Add(0, 1);
}
else if (winner != playerNumber && winner != 0) {
if (!Q[newState.CustomHash()].ContainsKey(0))
Q[newState.CustomHash()].Add(0, -1);
}
N[prevState.CustomHash()][prevAction.CustomHash()]++;
Q[prevState.CustomHash()][prevAction.CustomHash()] =
Q[prevState.CustomHash()][prevAction.CustomHash()]
+ LearningRate(N[prevState.CustomHash()][prevAction.CustomHash()])
* (prevReward + discount * GetMaxQ(newState) - Q[prevState.CustomHash()][prevAction.CustomHash()]);
}
prevState = newState;
prevAction = GetMaxExplorationFunctionA(newState);
prevReward = 0;
return prevAction;
}
public override Move Move(Virus percept) {
VirusBoard state = percept.GetBoardCopy();
Move[] actions = state.GetPossibleMoves(playerNumber);
if (actions.Length < 1)
return default(Move);
Move action;
if (random.NextDouble() > randomRatio) { // bruteforce
List<Move> list = new List<Move>();
int maxtaken = -1;
foreach (Move a in actions) {
int temp = state.TakeablePieces(a);
if (a.IsLongMove)
temp--;
if (temp > maxtaken) {
maxtaken = temp;
list.Clear();
list.Add(a);
}
else if (temp == maxtaken) {
list.Add(a);
}
}
if (deterministic) {
action = list[0];
}
else {
action = list[random.Next(list.Count)];
}
}
else { // random
if (deterministic)
action = actions[0];
else
action = actions[random.Next(actions.Length)];
}
return action;
}
public override void EndGame(Virus percept)
{
if (learn)
{
double reward = 0;
byte winner = percept.Winner;
if (winner == playerNumber)
{
reward = 1;
}
else if (winner != playerNumber && winner != 0)
{
reward = -1;
}
else
{
reward = 0;
}
if (!N.ContainsKey(prevState.CustomHash()))
{
N.Add(prevState.CustomHash(), new Dictionary <UInt32, int>());
}
if (!N[prevState.CustomHash()].ContainsKey(prevAction.CustomHash()))
{
N[prevState.CustomHash()].Add(prevAction.CustomHash(), 0);
}
N[prevState.CustomHash()][prevAction.CustomHash()]++;
Q[prevState.CustomHash()][prevAction.CustomHash()] =
Q[prevState.CustomHash()][prevAction.CustomHash()]
+ LearningRate(N[prevState.CustomHash()][prevAction.CustomHash()])
* (reward - Q[prevState.CustomHash()][prevAction.CustomHash()]);
}
prevState = default(VirusBoard);
prevAction = default(Move);
prevReward = 0;
}
public override Move Move(Virus percept)
{
VirusBoard newState = percept.GetBoardCopy();
if (!Q.ContainsKey(newState.CustomHash()))
{
Q.Add(newState.CustomHash(), new Dictionary <UInt32, double>());
}
if (learn && !prevState.Equals(default(VirusBoard)))
{
if (!N.ContainsKey(prevState.CustomHash()))
{
N.Add(prevState.CustomHash(), new Dictionary <UInt32, int>());
}
if (!N[prevState.CustomHash()].ContainsKey(prevAction.CustomHash()))
{
N[prevState.CustomHash()].Add(prevAction.CustomHash(), 0);
}
N[prevState.CustomHash()][prevAction.CustomHash()]++;
Q[prevState.CustomHash()][prevAction.CustomHash()] =
Q[prevState.CustomHash()][prevAction.CustomHash()]
+ LearningRate(N[prevState.CustomHash()][prevAction.CustomHash()])
* (prevReward + discount * GetMaxQ(newState) - Q[prevState.CustomHash()][prevAction.CustomHash()]);
}
prevState = newState;
prevAction = GetMaxExplorationFunctionA(newState);
prevReward = 0;
if (learn && !Q[prevState.CustomHash()].ContainsKey(prevAction.CustomHash()))
{
Q[prevState.CustomHash()].Add(prevAction.CustomHash(), initvalue);
}
return(prevAction);
}
public abstract void EndGame(Virus percept);
public abstract Move Move(Virus percept);
static byte RunGame(Virus virus, Agent p1, Agent p2) {
Move move;
byte winner;
while (true) {
move = p1.Move(virus);
virus.Move(move.sx, move.sy, move.ex, move.ey);
winner = virus.Winner;
if (winner != 0)
break;
move = p2.Move(virus);
virus.Move(move.sx, move.sy, move.ex, move.ey);
winner = virus.Winner;
if (winner != 0)
break;
}
p1.EndGame(virus);
p2.EndGame(virus);
return winner;
}
static void ShowBoard(Virus virus) {
for (int i = virus.Size - 1; i >= 0; i--) {
for (int j = 0; j < virus.Size; j++) {
Console.Write("{0} ", virus[j, i]);
}
Console.WriteLine();
}
}
static void TestGame() {
bool testresult = false;
Virus virus = new Virus();
ShowBoard(virus);
testresult = ShouldCrashTest(() => virus.Move(0, 4, 0, 4)); //moving starting player's piece to it's own position
ShowResult("Test 1", testresult);
testresult = ShouldCrashTest(() => virus.Move(0, 4, 0, 1)); //moving starting player's piece too far
ShowResult("Test 2", testresult);
testresult = ShouldCrashTest(() => virus.Move(0, 0, 0, 1)); //moving non-starting player's piece
ShowResult("Test 3", testresult);
testresult = ShouldCrashTest(() => virus.Move(0, 0, 0, 3)); //moving non-starting player's piece too far
ShowResult("Test 4", testresult);
Console.WriteLine("The game should be in the start state:");
ShowBoard(virus);
Console.WriteLine("Making a move:\nThis should move the top left piece one space SE");
virus.Move(0, 4, 1, 3);
ShowBoard(virus);
testresult = virus.Winner == 0; //there shouldn't be a winner yet
ShowResult("Test winner", testresult);
testresult = ShouldCrashTest(() => virus.Move(1, 3, 0, 2)); //moving player 1's piece again
ShowResult("Test own piece", testresult);
testresult = ShouldCrashTest(() => virus.Move(0, 0, 0, 3)); //moving player 2's piece too far
ShowResult("Test opponent's piece", testresult);
Console.WriteLine("Making a move:\nThis should move the bottom left piece one space N");
virus.Move(0, 0, 0, 1);
ShowBoard(virus);
testresult = virus.Winner == 0; //there shouldn't be a winner yet
ShowResult("Test winner", testresult);
testresult = ShouldCrashTest(() => virus.Move(0, 4, 3, 0)); //moving player 1's piece too far
ShowResult("Test own piece", testresult);
testresult = ShouldCrashTest(() => virus.Move(0, 0, 0, 2)); //moving player 2's piece again
ShowResult("Test opponent's piece", testresult);
Console.WriteLine("Making a move:\nThis should move the piece at (1,3) two spaces S\n and take the adjacent pieces");
virus.Move(1, 3, 1, 1);
ShowBoard(virus);
testresult = virus.Winner == 0; //there shouldn't be a winner yet
ShowResult("Test winner", testresult);
testresult = ShouldCrashTest(() => virus.Move(1, 1, 1, 0)); //moving player 1's piece again
ShowResult("Test own piece", testresult);
testresult = ShouldCrashTest(() => virus.Move(4, 4, 1, 0)); //moving player 2's piece too far
ShowResult("Test opponent's piece", testresult);
}
static void TrainMadQ(byte qnumber, QAgent agent, String logname, String savename, int iterations) {
Virus virus = new Virus(2, 5);
int wins = 0, wins2 = 0;
byte oppnumber = qnumber == 1 ? (byte)2 : (byte)1;
Agent opp = new MixedAgent(0.1, false, oppnumber);
StreamWriter writer = new StreamWriter(logname);
for (int i = 1; i <= iterations; i++) {
int winner = RunGame(virus, qnumber == 1 ? (Agent)agent : opp, qnumber == 2 ? (Agent)agent : opp);
wins += winner == 1 ? 1 : 0;
wins2 += winner == 1 ? 1 : 0;
if (i % 100 == 0) {
writer.WriteLine(wins2);
wins2 = 0;
}
if (i % 10000 == 0) {
agent.Save(savename);
Console.WriteLine("Iteration: " + i);
Console.WriteLine("Wins: " + wins);
wins = 0;
}
virus = new Virus(2, 5);
}
writer.Close();
}
public VirusInterface(Virus virus, int tilesize = 20, bool immediateAI = false, params String[] names) {
InitializeComponent();
this.virus = virus;
this.tileSize = tilesize;
this.immediateAI = immediateAI;
this.MouseClick += MouseClickHandler1;
this.Size = new Size(
virus.Size * tileSize + 17,
virus.Size * tileSize + 55);
this.names.Add("Player 0");
this.names.AddRange(names);
while (this.names.Count < virus.Players + 1) {
this.names.Add("Player " + this.names.Count);
}
//Save("Lalalafil");
agents = new Agent[this.names.Count];
int n = 1;
for (byte i = 1; i < this.names.Count; i++) {
String p = this.names[i];
switch (p) {
case "QAI":
agents[i] = new QAgent(i);
if (File.Exists("TrainingData.Q") && File.Exists("TrainingData.N")) {
((QAgent)agents[i]).Load("TrainingData");
((QAgent)agents[i]).TurnOffExploration();
((QAgent)agents[i]).TurnOffLearning();
}
this.names[i] = "AI " + n;
n++;
break;
case "AnnAI":
agents[i] = new AnnAgent(false, virus.Size, i);
this.names[i] = "AI " + n;
n++;
break;
case "MinimaxAI":
agents[i] = new MinimaxAgent(4,i);
this.names[i] = "AI " + n;
n++;
break;
case "MiniMaxMixAI":
if (File.Exists("TrainingData.Q"))
agents[i] = new MiniMaxMixAgent("TrainingData", 2, i);
else
agents[i] = new BruteForceAgent(i);
this.names[i] = "AI " + n;
n++;
break;
case "MixedAI":
agents[i] = new MixedAgent(0.5,false,i);
this.names[i] = "AI " + n;
n++;
break;
case "BruteAI":
agents[i] = new BruteForceAgent(i);
this.names[i] = "AI " + n;
n++;
break;
case "RandomAI":
agents[i] = new RandomAgent(i);
this.names[i] = "AI " + n;
n++;
break;
case "SimpleAI":
agents[i] = new SimpleAgent(i);
this.names[i] = "AI " + n;
n++;
break;
}
}
message = this.names[1] + "'s turn";
colors = new Color[virus.Players + 1];
colors[0] = Color.White;
colors[1] = Color.FromArgb(128, 160, 255);
colors[2] = Color.FromArgb(96, 255, 96);
if(virus.Players > 2)
colors[3] = Color.FromArgb(255, 96, 96);
if(virus.Players > 3)
colors[4] = Color.FromArgb(255, 255, 64);
Random rand = new Random();
for (int i = 5; i <= virus.Players; i++)
colors[i] = Color.FromArgb(rand.Next(256), rand.Next(256), rand.Next(256));
}
public void StartGame(
Virus virus,
PerformedMoveCallback callback,
UpdatePiecesCallback piecesCallback,
EndCallback end,
string id,
params VirusPlayer[] players)
{
Random rand = new Random();
PerformedMove = callback;
UpdatePieces = piecesCallback;
End = end;
PlayerID = id;
this.virus = virus;
this.immediateAI = true;
this.MouseClick += MouseClickHandler1;
tileSize = 49;
this.Size = new Size(
virus.Size * tileSize + 17,
virus.Size * tileSize + 55);
int smallestSide = this.Size.Height < this.Size.Width ? this.Size.Height : this.Size.Width;
tileSize = smallestSide / virus.Size;
this.players.Add(new VirusPlayer("Player 0", "", Color.White));
this.players.AddRange(players);
while (this.players.Count < virus.Players + 1) {
this.players.Add(new VirusPlayer("BruteAI","AI",Color.FromArgb(rand.Next(256), rand.Next(256), rand.Next(256))));
}
//Save("Lalalafil");
agents = new Agent[this.players.Count];
int n = 1;
for (byte i = 1; i < this.players.Count; i++) {
String p = this.players[i].Name;
switch (p) {
case "AIQ":
agents[i] = new QAgent(i);
if (File.Exists("TrainingData.Q") && File.Exists("TrainingData.N")) {
((QAgent)agents[i]).Load("TrainingData");
((QAgent)agents[i]).TurnOffExploration();
((QAgent)agents[i]).TurnOffLearning();
}
//this.players[i].Name = "AI " + n;
n++;
break;
case "AIMQ":
agents[i] = new MemoryQAgent(i);
if (File.Exists("TrainingData.Q") && File.Exists("TrainingData.N")) {
((MemoryQAgent)agents[i]).Load("TrainingData");
((MemoryQAgent)agents[i]).TurnOffExploration();
}
//this.players[i].Name = "AI " + n;
n++;
break;
case "AIMinimax":
agents[i] = new MinimaxAgent(4,i);
//this.players[i].Name = "AI " + n;
n++;
break;
case "AIMiniMaxMix":
if (File.Exists("TrainingData.Q"))
agents[i] = new MiniMaxMixAgent("TrainingData", 2, i);
else
agents[i] = new BruteForceAgent(i);
//this.players[i].Name = "AI " + n;
n++;
break;
case "AIMixed":
agents[i] = new MixedAgent(0.5,false,i);
//this.players[i].Name = "AI " + n;
n++;
break;
case "AIBrute":
agents[i] = new BruteForceAgent(i);
//this.players[i].Name = "AI " + n;
n++;
break;
case "AIRandom":
agents[i] = new RandomAgent(i);
//this.players[i].Name = "AI " + n;
n++;
break;
case "AISimple":
agents[i] = new SimpleAgent(i);
//this.players[i].Name = "AI " + n;
n++;
break;
}
}
message = this.players[1].Name + "'s turn";
/*colors = new Color[virus.Players + 1];
colors[0] = Color.White;
colors[1] = Color.FromArgb(128, 160, 255);
colors[2] = Color.FromArgb(96, 255, 96);
if(virus.Players > 2)
colors[3] = Color.FromArgb(255, 96, 96);
if(virus.Players > 3)
colors[4] = Color.FromArgb(255, 255, 64);
for (int i = 5; i <= virus.Players; i++)
colors[i] = Color.FromArgb(rand.Next(256), rand.Next(256), rand.Next(256));*/
}
public override void EndGame(Virus percept)
{
}
public abstract Move Move(Virus percept);
public override void EndGame(Virus percept) {
}
public override Move Move(Virus percept) {
VirusBoard newState = percept.GetBoardCopy();
if (!Q.ContainsKey(newState.CustomHash()))
Q.Add(newState.CustomHash(), new Dictionary<UInt32, double>());
if (learn && !prevState.Equals(default(VirusBoard))) {
if (!N.ContainsKey(prevState.CustomHash()))
N.Add(prevState.CustomHash(), new Dictionary<UInt32, int>());
if (!N[prevState.CustomHash()].ContainsKey(prevAction.CustomHash()))
N[prevState.CustomHash()].Add(prevAction.CustomHash(), 0);
N[prevState.CustomHash()][prevAction.CustomHash()]++;
Q[prevState.CustomHash()][prevAction.CustomHash()] =
Q[prevState.CustomHash()][prevAction.CustomHash()]
+ LearningRate(N[prevState.CustomHash()][prevAction.CustomHash()])
* (prevReward + discount * GetMaxQ(newState) - Q[prevState.CustomHash()][prevAction.CustomHash()]);
}
prevState = newState;
prevAction = GetMaxExplorationFunctionA(newState);
prevReward = 0;
if (learn && !Q[prevState.CustomHash()].ContainsKey(prevAction.CustomHash()))
Q[prevState.CustomHash()].Add(prevAction.CustomHash(), initvalue);
return prevAction;
}
private void ImmediateHandler() {
immediateRunning = true;
while (names[virus.CurrentPlayer].StartsWith("AI")) {
AIMove();
if (gameWon) {
foreach (Agent a in agents) {
if (a != null) {
a.EndGame(virus);
}
}
gameWon = false;
virus = new Virus(virus.Players, virus.Size);
this.Refresh();
}
if (!immediateAI)
return;
}
immediateRunning = false;
}
static void comTestMemQ2() {
//train a MemQ agent, and once every ten games have it teach an observer
//do this with ten agents
const int teachingagents = 10;
const int gamesbetweenteaching = 10;
const int learninggames = 10000;
int wins = 0;
int n = 0;
while (File.Exists("observer" + n + ".log")) n++;
StreamWriter writer = new StreamWriter("observer" + n + ".log");
writer.WriteLine("sources");
Virus virus;
MemoryQAgent observer = new MemoryQAgent(1);
Agent opponent = new BruteForceAgent(2);
MemoryQAgent[] agents = new MemoryQAgent[teachingagents];
for (int i = 0; i < teachingagents; i++) {
agents[i] = new MemoryQAgent(1);
wins = 0;
for (int j = 0; j < learninggames; j++) {
virus = new Virus(2, 5);
wins += RunGame(virus, agents[i], opponent) == 1 ? 1 : 0;
agents[i].ProcessShortTermMemory();
if (j % 1000 == 999) {
if (j > 9500) {
writer.WriteLine(wins);
Console.WriteLine("Wins: " + wins);
}
wins = 0;
}
}
}
writer.WriteLine("observer");
int totalmems = 0;
int memstoprocess = 0;
while (totalmems < 100000) {
//if (memstoprocess == 10000) memstoprocess = 100000;
//if (memstoprocess == 1000) memstoprocess = 10000;
//if (memstoprocess < 1000) memstoprocess = 1000;
memstoprocess += 1000;
while (totalmems < memstoprocess) {
for (int i = 0; i < teachingagents; i++) {
totalmems += agents[i].TellOfMemoryToExt(observer, true);
}
}
//writer.WriteLine("observer");
//for (int i = 0; i < 10; i++) {
wins = 0;
for (int j = 0; j < 10000; j++) {
virus = new Virus();
wins += RunGame(virus, observer, opponent) == 1 ? 1 : 0;
observer.ForgetShortTerm();
}
writer.WriteLine(wins);
Console.WriteLine("Observer wins after " + totalmems + "memories: " + wins);
//}
}
writer.Close();
}
static void TrainQ(int size, byte qnumber, QAgent agent, String opponent, String logname, String savename, int iterations, int saveinterval = 360) {
Virus virus = new Virus(2, size);
int wins = 0, wins2 = 0;
byte oppnumber = qnumber == 1 ? (byte)2 : (byte)1;
Agent opp = new BruteForceAgent(oppnumber);
StreamWriter writer = new StreamWriter(logname);
for (int i = 1; i <= iterations; i++) {
switch (opponent) {
case "brute":
break;
case "minimax4":
opp = new MinimaxAgent(4, oppnumber);
break;
case "minimax3":
opp = new MinimaxAgent(3, oppnumber);
break;
case "minimax2":
opp = new MinimaxAgent(2, oppnumber);
break;
default:
opp = new BruteForceAgent(oppnumber);
break;
}
int winner = RunGame(virus, qnumber == 1 ? (Agent)agent : opp, qnumber == 2 ? (Agent)agent : opp);
wins += winner == 1 ? 1 : 0;
wins2 += winner == 1 ? 1 : 0;
if (i % 100 == 0) {
if (agent.RandomRate == 0) {
writer.WriteLine(wins2);
}
wins2 = 0;
}
if (i % saveinterval == 0) {
agent.Save(savename);
Console.WriteLine("Iteration: " + i);
Console.WriteLine("Wins: " + wins);
wins = 0;
if (agent.RandomRate > 0) {
agent.TurnOffExploration();
agent.TurnOffLearning();
for (int j = 1; j <= 1000; j++) {
virus = new Virus(2, size);
winner = RunGame(virus, qnumber == 1 ? (Agent)agent : opp, qnumber == 2 ? (Agent)agent : opp);
wins += winner == 1 ? 1 : 0;
}
writer.WriteLine(wins);
wins = 0;
agent.TurnOnExploration();
agent.TurnOnLearning();
}
}
virus = new Virus(2, size);
}
writer.Close();
}
private void MouseClickHandler1(Object sender, MouseEventArgs args) {
if (names[virus.CurrentPlayer].StartsWith("AI")) {
if (!immediateAI) {
if ((args.X > this.Width - 60) && (args.Y > virus.Size * tileSize)) {
AIMove();
}
}
}
else if (!immediateRunning) {
PlayerMove(args);
}
if (gameWon) {
foreach (Agent a in agents) {
if (a != null) {
a.EndGame(virus);
}
}
gameWon = false;
virus = new Virus(virus.Players, virus.Size);
this.Refresh();
}
if (immediateAI && !immediateRunning) {
ImmediateHandler();
}
}
static void TrainMemoryQ(int size, byte qnumber, MemoryQAgent agent, String opponent, String logname, String savename, int iterations, int saveinterval = 360) {
Virus virus = new Virus(2, size);
int wins = 0, wins2 = 0;
byte oppnumber = qnumber == 1 ? (byte)2 : (byte)1;
Agent opp = new BruteForceAgent(oppnumber);
StreamWriter writer = new StreamWriter(logname);
for (int i = 1; i <= iterations; i++) {
switch (opponent) {
case "brute":
break;
case "minimax4":
opp = new MinimaxAgent(4, oppnumber);
break;
case "minimax3":
opp = new MinimaxAgent(3, oppnumber);
break;
case "minimax2":
opp = new MinimaxAgent(2, oppnumber);
break;
default:
opp = new BruteForceAgent(oppnumber);
break;
}
int winner = RunGame(virus, qnumber == 1 ? (Agent)agent : opp, qnumber == 2 ? (Agent)agent : opp);
wins += winner == 1 ? 1 : 0;
wins2 += winner == 1 ? 1 : 0;
agent.ProcessShortTermMemory();
if (i % saveinterval == 0) {
agent.Save(savename);
Console.WriteLine("Iteration: " + i);
Console.WriteLine("Wins: " + wins);
wins = 0;
}
virus = new Virus(2, size);
}
for (int n = 0; n < 10; n++) {
MemoryQAgent ag = new MemoryQAgent();
for (int i = 0; i < 1000; i++) {
agent.TellOfMemoryTo(ag);
}
wins = 0;
opp = new BruteForceAgent(oppnumber);
for (int i = 1; i <= 10000; i++) {
switch (opponent) {
case "brute":
break;
case "minimax4":
opp = new MinimaxAgent(4, oppnumber);
break;
case "minimax3":
opp = new MinimaxAgent(3, oppnumber);
break;
case "minimax2":
opp = new MinimaxAgent(2, oppnumber);
break;
default:
opp = new BruteForceAgent(oppnumber);
break;
}
int winner = RunGame(virus, qnumber == 1 ? (Agent)ag : opp, qnumber == 2 ? (Agent)ag : opp);
wins += winner == 1 ? 1 : 0;
virus = new Virus(2, size);
}
Console.WriteLine("After 1000 memories");
Console.WriteLine("Wins: " + wins);
writer.Write(((double)wins) / 10000.0 + ";");
for (int i = 0; i < 9000; i++) {
agent.TellOfMemoryTo(ag);
}
wins = 0;
opp = new BruteForceAgent(oppnumber);
for (int i = 1; i <= 10000; i++) {
switch (opponent) {
case "brute":
break;
case "minimax4":
opp = new MinimaxAgent(4, oppnumber);
break;
case "minimax3":
opp = new MinimaxAgent(3, oppnumber);
break;
case "minimax2":
opp = new MinimaxAgent(2, oppnumber);
break;
default:
opp = new BruteForceAgent(oppnumber);
break;
}
int winner = RunGame(virus, qnumber == 1 ? (Agent)ag : opp, qnumber == 2 ? (Agent)ag : opp);
wins += winner == 1 ? 1 : 0;
virus = new Virus(2, size);
}
Console.WriteLine("After 10000 memories");
Console.WriteLine("Wins: " + wins);
writer.Write(((double)wins) / 10000.0 + ";");
for (int i = 0; i < 90000; i++) {
agent.TellOfMemoryTo(ag);
}
wins = 0;
opp = new BruteForceAgent(oppnumber);
for (int i = 1; i <= 10000; i++) {
switch (opponent) {
case "brute":
break;
case "minimax4":
opp = new MinimaxAgent(4, oppnumber);
break;
case "minimax3":
opp = new MinimaxAgent(3, oppnumber);
break;
case "minimax2":
opp = new MinimaxAgent(2, oppnumber);
break;
default:
opp = new BruteForceAgent(oppnumber);
break;
}
int winner = RunGame(virus, qnumber == 1 ? (Agent)ag : opp, qnumber == 2 ? (Agent)ag : opp);
wins += winner == 1 ? 1 : 0;
virus = new Virus(2, size);
}
Console.WriteLine("After 100000 memories");
Console.WriteLine("Wins: " + wins);
writer.WriteLine(((double)wins) / 10000.0);
}
writer.Close();
agent.SaveLongTermMemory("m");
}
static void RunMinimax(int size, byte qnumber, MinimaxAgent agent, String opponent, int iterations, int saveinterval = 360) {
Virus virus = new Virus(2, size);
int wins = 0;
int wins2 = 0;
byte oppnumber = qnumber == 1 ? (byte)2 : (byte)1;
int n = 0;
while (File.Exists("mmlog" + n)) {
n++;
}
StreamWriter writer = new StreamWriter("mmlog" + n);
StreamWriter timeWriter = new StreamWriter("mmTimeLog", true);
timeWriter.WriteLine("Player number: {0} Board size: {1} Opponent: {2}", qnumber, size, opponent);
timeWriter.Close();
for (int i = 1; i <= iterations; i++) {
Agent opp;
switch (opponent) {
case "brute":
opp = new BruteForceAgent(oppnumber);
break;
case "minimax4":
opp = new MinimaxAgent(4, oppnumber);
break;
case "minimax3":
opp = new MinimaxAgent(3, oppnumber);
break;
case "minimax2":
opp = new MinimaxAgent(2, oppnumber);
break;
default:
opp = new BruteForceAgent(oppnumber);
break;
}
virus.SaveReplay = true;
int winner = RunGame(virus, qnumber == 1 ? (Agent)agent : opp, qnumber == 2 ? (Agent)agent : opp);
wins += winner == 1 ? 1 : 0;
wins2 += winner == 1 ? 1 : 0;
/*if (i % 100 == 0) {
writer.WriteLine(wins2);
//Console.WriteLine("Iteration: " + i);
//Console.WriteLine("Wins: " + wins);
wins2 = 0;
}*/
if (i % saveinterval == 0) {
writer.WriteLine(wins);
Console.WriteLine("Iteration: " + i);
Console.WriteLine("Wins: " + wins);
//Console.WriteLine(winners);
//winners.Clear();
wins = 0;
}
virus = new Virus(2, size);
}
writer.Close();
}
public abstract void EndGame(Virus percept);
static void RunQ(byte qnumber, QAgent agent, String opponent, int iterations, int size) {
Virus virus = new Virus(2, size);
int wins = 0;
byte oppnumber = qnumber == 1 ? (byte)2 : (byte)1;
Agent opp;
switch (opponent) {
case "brute":
opp = new BruteForceAgent(oppnumber);
break;
case "minimax4":
opp = new MinimaxAgent(4, oppnumber);
break;
case "minimax3":
opp = new MinimaxAgent(3, oppnumber);
break;
case "minimax2":
opp = new MinimaxAgent(2, oppnumber);
break;
default:
opp = new BruteForceAgent(oppnumber);
break;
}
int n = 0;
while (File.Exists("qwinlog" + n)) {
n++;
}
StreamWriter writer = new StreamWriter("qwinlog" + n);
for (int i = 1; i <= iterations; i++) {
int winner = RunGame(virus, qnumber == 1 ? (Agent)agent : opp, qnumber == 2 ? (Agent)agent : opp);
wins += winner == 1 ? 1 : 0;
virus = new Virus(2, size);
}
writer.WriteLine(wins);
Console.WriteLine("Iteration: " + iterations);
Console.WriteLine("Wins: " + wins);
wins = 0;
writer.Close();
}
public override void EndGame(Virus percept) {
Move(percept);
prevState = default(VirusBoard);
prevAction = default(Move);
prevReward = 0;
}
private void MouseClickHandler1(Object sender, MouseEventArgs args) {
if (players[virus.CurrentPlayer].Name.StartsWith("AI")) {
if (!immediateAI) {
if ((args.X > this.Width - 60) && (args.Y > virus.Size * tileSize)) {
if (players[virus.CurrentPlayer].ID == PlayerID) {
AINetworkMove();
}
}
}
}
else if (!immediateRunning) {
int tileX = args.X / tileSize;
int tileY = virus.Size - (args.Y / tileSize) - 1;
PlayerMove(tileX, tileY);
}
if (gameWon) {
foreach (Agent a in agents) {
if (a != null) {
a.EndGame(virus);
}
}
gameWon = false;
virus = new Virus(virus.Players, virus.Size);
this.Refresh();
}
if (immediateAI && !immediateRunning) {
ImmediateHandler();
}
}