public int TellOfMemoryToExt(MemoryQAgent agent, bool fullepisode = false) { VirusMemory[] memories = LongTermMemory[random.Next(LongTermMemory.Count)].Memories; VirusMemory memory = memories[0]; if (fullepisode) { agent.LearnFromEpisode(memories, true); //foreach (VirusMemory m in memories) // agent.Learn(m); return(memories.Length); } else { foreach (VirusMemory m in memories) { if (Math.Abs(m.Reward) > Math.Abs(memory.Reward)) { memory = m; } } agent.Learn(memory); return(1); } }
static void comTestMemQ() { //train a MemQ agent, and once every ten games have it teach an observer //do this with ten agents const int teachingagents = 10; const int gamesbetweenteaching = 10; const int learninggames = 10000; int wins = 0; int n = 0; while (File.Exists("observer" + n + ".log")) n++; StreamWriter writer = new StreamWriter("observer" + n + ".log"); writer.WriteLine("sources"); Virus virus; MemoryQAgent observer = new MemoryQAgent(1); Agent opponent = new BruteForceAgent(2); for (int i = 0; i < teachingagents; i++) { MemoryQAgent agent = new MemoryQAgent(1); wins = 0; for (int j = 0; j < learninggames; j++) { virus = new Virus(2, 5); wins += RunGame(virus, agent, opponent) == 1 ? 1 : 0; agent.ProcessShortTermMemory(); if (j % gamesbetweenteaching == gamesbetweenteaching - 1) { agent.TellOfMemoryTo(observer, true); } if (j % 1000 == 999) { if (j > 9500) { writer.WriteLine(wins); Console.WriteLine("Wins: " + wins); } wins = 0; } } } writer.WriteLine("observer"); for (int i = 0; i < 10; i++) { wins = 0; for (int j = 0; j < 1000; j++) { virus = new Virus(); wins += RunGame(virus, observer, opponent) == 1 ? 1 : 0; observer.ForgetShortTerm(); } writer.WriteLine(wins); Console.WriteLine("Observer wins: " + wins); } writer.Close(); }
static void GatherMemQData(String opponent, int iterations, int saveinterval, byte player) { MemoryQAgent agent; String name = opponent + "MemQ"; Console.WriteLine("Gathering data for Q: " + name + ":"); for (int i = 0; i < 10; i++) { if (!File.Exists(name + ".log" + i)) { agent = new MemoryQAgent(player); TrainMemoryQ(5, player, agent, opponent, name + ".log" + i, name + ".sav" + i, iterations, saveinterval); } } }
static void comTrainMemQ(bool load) { Console.WriteLine("Q is player..?"); byte temp3 = byte.Parse(Console.ReadLine()); MemoryQAgent mqagent = new MemoryQAgent(temp3); if (load) mqagent.Load("TrainingData"); Console.WriteLine("How many iterations?"); int temp1 = int.Parse(Console.ReadLine()); Console.WriteLine("Save how often?"); int temp2 = int.Parse(Console.ReadLine()); Console.WriteLine("Board size?"); int temp4 = int.Parse(Console.ReadLine()); Console.WriteLine("Which opponent?"); string command = Console.ReadLine(); TrainMemoryQ(temp4, temp3, mqagent, command, "log", "TrainingData", temp1, temp2); }
public void TellOfMemoryTo(MemoryQAgent agent, bool fullepisode = false) { VirusMemory[] memories = LongTermMemory[random.Next(LongTermMemory.Count)].Memories; VirusMemory memory = memories[0]; if (fullepisode) { foreach (VirusMemory m in memories) { agent.Learn(m); } } else { foreach (VirusMemory m in memories) { if (Math.Abs(m.Reward) > Math.Abs(memory.Reward)) { memory = m; } } agent.Learn(memory); } }
static void TrainMemoryQ(int size, byte qnumber, MemoryQAgent agent, String opponent, String logname, String savename, int iterations, int saveinterval = 360) { Virus virus = new Virus(2, size); int wins = 0, wins2 = 0; byte oppnumber = qnumber == 1 ? (byte)2 : (byte)1; Agent opp = new BruteForceAgent(oppnumber); StreamWriter writer = new StreamWriter(logname); for (int i = 1; i <= iterations; i++) { switch (opponent) { case "brute": break; case "minimax4": opp = new MinimaxAgent(4, oppnumber); break; case "minimax3": opp = new MinimaxAgent(3, oppnumber); break; case "minimax2": opp = new MinimaxAgent(2, oppnumber); break; default: opp = new BruteForceAgent(oppnumber); break; } int winner = RunGame(virus, qnumber == 1 ? (Agent)agent : opp, qnumber == 2 ? (Agent)agent : opp); wins += winner == 1 ? 1 : 0; wins2 += winner == 1 ? 1 : 0; agent.ProcessShortTermMemory(); if (i % saveinterval == 0) { agent.Save(savename); Console.WriteLine("Iteration: " + i); Console.WriteLine("Wins: " + wins); wins = 0; } virus = new Virus(2, size); } for (int n = 0; n < 10; n++) { MemoryQAgent ag = new MemoryQAgent(); for (int i = 0; i < 1000; i++) { agent.TellOfMemoryTo(ag); } wins = 0; opp = new BruteForceAgent(oppnumber); for (int i = 1; i <= 10000; i++) { switch (opponent) { case "brute": break; case "minimax4": opp = new MinimaxAgent(4, oppnumber); break; case "minimax3": opp = new MinimaxAgent(3, oppnumber); break; case "minimax2": opp = new MinimaxAgent(2, oppnumber); break; default: opp = new BruteForceAgent(oppnumber); break; } int winner = RunGame(virus, qnumber == 1 ? (Agent)ag : opp, qnumber == 2 ? (Agent)ag : opp); wins += winner == 1 ? 1 : 0; virus = new Virus(2, size); } Console.WriteLine("After 1000 memories"); Console.WriteLine("Wins: " + wins); writer.Write(((double)wins) / 10000.0 + ";"); for (int i = 0; i < 9000; i++) { agent.TellOfMemoryTo(ag); } wins = 0; opp = new BruteForceAgent(oppnumber); for (int i = 1; i <= 10000; i++) { switch (opponent) { case "brute": break; case "minimax4": opp = new MinimaxAgent(4, oppnumber); break; case "minimax3": opp = new MinimaxAgent(3, oppnumber); break; case "minimax2": opp = new MinimaxAgent(2, oppnumber); break; default: opp = new BruteForceAgent(oppnumber); break; } int winner = RunGame(virus, qnumber == 1 ? (Agent)ag : opp, qnumber == 2 ? (Agent)ag : opp); wins += winner == 1 ? 1 : 0; virus = new Virus(2, size); } Console.WriteLine("After 10000 memories"); Console.WriteLine("Wins: " + wins); writer.Write(((double)wins) / 10000.0 + ";"); for (int i = 0; i < 90000; i++) { agent.TellOfMemoryTo(ag); } wins = 0; opp = new BruteForceAgent(oppnumber); for (int i = 1; i <= 10000; i++) { switch (opponent) { case "brute": break; case "minimax4": opp = new MinimaxAgent(4, oppnumber); break; case "minimax3": opp = new MinimaxAgent(3, oppnumber); break; case "minimax2": opp = new MinimaxAgent(2, oppnumber); break; default: opp = new BruteForceAgent(oppnumber); break; } int winner = RunGame(virus, qnumber == 1 ? (Agent)ag : opp, qnumber == 2 ? (Agent)ag : opp); wins += winner == 1 ? 1 : 0; virus = new Virus(2, size); } Console.WriteLine("After 100000 memories"); Console.WriteLine("Wins: " + wins); writer.WriteLine(((double)wins) / 10000.0); } writer.Close(); agent.SaveLongTermMemory("m"); }
public int TellOfMemoryToExt(MemoryQAgent agent, bool fullepisode = false) { VirusMemory[] memories = LongTermMemory[random.Next(LongTermMemory.Count)].Memories; VirusMemory memory = memories[0]; if (fullepisode) { agent.LearnFromEpisode(memories, true); //foreach (VirusMemory m in memories) // agent.Learn(m); return memories.Length; } else { foreach (VirusMemory m in memories) if (Math.Abs(m.Reward) > Math.Abs(memory.Reward)) memory = m; agent.Learn(memory); return 1; } }
public void TellOfMemoryTo(MemoryQAgent agent, bool fullepisode = false) { VirusMemory[] memories = LongTermMemory[random.Next(LongTermMemory.Count)].Memories; VirusMemory memory = memories[0]; if (fullepisode) { foreach (VirusMemory m in memories) agent.Learn(m); } else { foreach (VirusMemory m in memories) if (Math.Abs(m.Reward) > Math.Abs(memory.Reward)) memory = m; agent.Learn(memory); } }
static void comTestMemQ2() { //train a MemQ agent, and once every ten games have it teach an observer //do this with ten agents const int teachingagents = 10; const int gamesbetweenteaching = 10; const int learninggames = 10000; int wins = 0; int n = 0; while (File.Exists("observer" + n + ".log")) n++; StreamWriter writer = new StreamWriter("observer" + n + ".log"); writer.WriteLine("sources"); Virus virus; MemoryQAgent observer = new MemoryQAgent(1); Agent opponent = new BruteForceAgent(2); MemoryQAgent[] agents = new MemoryQAgent[teachingagents]; for (int i = 0; i < teachingagents; i++) { agents[i] = new MemoryQAgent(1); wins = 0; for (int j = 0; j < learninggames; j++) { virus = new Virus(2, 5); wins += RunGame(virus, agents[i], opponent) == 1 ? 1 : 0; agents[i].ProcessShortTermMemory(); if (j % 1000 == 999) { if (j > 9500) { writer.WriteLine(wins); Console.WriteLine("Wins: " + wins); } wins = 0; } } } writer.WriteLine("observer"); int totalmems = 0; int memstoprocess = 0; while (totalmems < 100000) { //if (memstoprocess == 10000) memstoprocess = 100000; //if (memstoprocess == 1000) memstoprocess = 10000; //if (memstoprocess < 1000) memstoprocess = 1000; memstoprocess += 1000; while (totalmems < memstoprocess) { for (int i = 0; i < teachingagents; i++) { totalmems += agents[i].TellOfMemoryToExt(observer, true); } } //writer.WriteLine("observer"); //for (int i = 0; i < 10; i++) { wins = 0; for (int j = 0; j < 10000; j++) { virus = new Virus(); wins += RunGame(virus, observer, opponent) == 1 ? 1 : 0; observer.ForgetShortTerm(); } writer.WriteLine(wins); Console.WriteLine("Observer wins after " + totalmems + "memories: " + wins); //} } writer.Close(); }
public void StartGame( Virus virus, PerformedMoveCallback callback, UpdatePiecesCallback piecesCallback, EndCallback end, string id, params VirusPlayer[] players) { Random rand = new Random(); PerformedMove = callback; UpdatePieces = piecesCallback; End = end; PlayerID = id; this.virus = virus; this.immediateAI = true; this.MouseClick += MouseClickHandler1; tileSize = 49; this.Size = new Size( virus.Size * tileSize + 17, virus.Size * tileSize + 55); int smallestSide = this.Size.Height < this.Size.Width ? this.Size.Height : this.Size.Width; tileSize = smallestSide / virus.Size; this.players.Add(new VirusPlayer("Player 0", "", Color.White)); this.players.AddRange(players); while (this.players.Count < virus.Players + 1) { this.players.Add(new VirusPlayer("BruteAI","AI",Color.FromArgb(rand.Next(256), rand.Next(256), rand.Next(256)))); } //Save("Lalalafil"); agents = new Agent[this.players.Count]; int n = 1; for (byte i = 1; i < this.players.Count; i++) { String p = this.players[i].Name; switch (p) { case "AIQ": agents[i] = new QAgent(i); if (File.Exists("TrainingData.Q") && File.Exists("TrainingData.N")) { ((QAgent)agents[i]).Load("TrainingData"); ((QAgent)agents[i]).TurnOffExploration(); ((QAgent)agents[i]).TurnOffLearning(); } //this.players[i].Name = "AI " + n; n++; break; case "AIMQ": agents[i] = new MemoryQAgent(i); if (File.Exists("TrainingData.Q") && File.Exists("TrainingData.N")) { ((MemoryQAgent)agents[i]).Load("TrainingData"); ((MemoryQAgent)agents[i]).TurnOffExploration(); } //this.players[i].Name = "AI " + n; n++; break; case "AIMinimax": agents[i] = new MinimaxAgent(4,i); //this.players[i].Name = "AI " + n; n++; break; case "AIMiniMaxMix": if (File.Exists("TrainingData.Q")) agents[i] = new MiniMaxMixAgent("TrainingData", 2, i); else agents[i] = new BruteForceAgent(i); //this.players[i].Name = "AI " + n; n++; break; case "AIMixed": agents[i] = new MixedAgent(0.5,false,i); //this.players[i].Name = "AI " + n; n++; break; case "AIBrute": agents[i] = new BruteForceAgent(i); //this.players[i].Name = "AI " + n; n++; break; case "AIRandom": agents[i] = new RandomAgent(i); //this.players[i].Name = "AI " + n; n++; break; case "AISimple": agents[i] = new SimpleAgent(i); //this.players[i].Name = "AI " + n; n++; break; } } message = this.players[1].Name + "'s turn"; /*colors = new Color[virus.Players + 1]; colors[0] = Color.White; colors[1] = Color.FromArgb(128, 160, 255); colors[2] = Color.FromArgb(96, 255, 96); if(virus.Players > 2) colors[3] = Color.FromArgb(255, 96, 96); if(virus.Players > 3) colors[4] = Color.FromArgb(255, 255, 64); for (int i = 5; i <= virus.Players; i++) colors[i] = Color.FromArgb(rand.Next(256), rand.Next(256), rand.Next(256));*/ }