public int TellOfMemoryToExt(MemoryQAgent agent, bool fullepisode = false)
{
VirusMemory[] memories = LongTermMemory[random.Next(LongTermMemory.Count)].Memories;
VirusMemory memory = memories[0];
if (fullepisode)
{
agent.LearnFromEpisode(memories, true);
//foreach (VirusMemory m in memories)
// agent.Learn(m);
return(memories.Length);
}
else
{
foreach (VirusMemory m in memories)
{
if (Math.Abs(m.Reward) > Math.Abs(memory.Reward))
{
memory = m;
}
}
agent.Learn(memory);
return(1);
}
}
static void comTestMemQ() {
//train a MemQ agent, and once every ten games have it teach an observer
//do this with ten agents
const int teachingagents = 10;
const int gamesbetweenteaching = 10;
const int learninggames = 10000;
int wins = 0;
int n = 0;
while (File.Exists("observer" + n + ".log")) n++;
StreamWriter writer = new StreamWriter("observer" + n + ".log");
writer.WriteLine("sources");
Virus virus;
MemoryQAgent observer = new MemoryQAgent(1);
Agent opponent = new BruteForceAgent(2);
for (int i = 0; i < teachingagents; i++) {
MemoryQAgent agent = new MemoryQAgent(1);
wins = 0;
for (int j = 0; j < learninggames; j++) {
virus = new Virus(2, 5);
wins += RunGame(virus, agent, opponent) == 1 ? 1 : 0;
agent.ProcessShortTermMemory();
if (j % gamesbetweenteaching == gamesbetweenteaching - 1) {
agent.TellOfMemoryTo(observer, true);
}
if (j % 1000 == 999) {
if (j > 9500) {
writer.WriteLine(wins);
Console.WriteLine("Wins: " + wins);
}
wins = 0;
}
}
}
writer.WriteLine("observer");
for (int i = 0; i < 10; i++) {
wins = 0;
for (int j = 0; j < 1000; j++) {
virus = new Virus();
wins += RunGame(virus, observer, opponent) == 1 ? 1 : 0;
observer.ForgetShortTerm();
}
writer.WriteLine(wins);
Console.WriteLine("Observer wins: " + wins);
}
writer.Close();
}
static void GatherMemQData(String opponent, int iterations, int saveinterval, byte player) {
MemoryQAgent agent;
String name = opponent + "MemQ";
Console.WriteLine("Gathering data for Q: " + name + ":");
for (int i = 0; i < 10; i++) {
if (!File.Exists(name + ".log" + i)) {
agent = new MemoryQAgent(player);
TrainMemoryQ(5, player, agent, opponent, name + ".log" + i, name + ".sav" + i, iterations, saveinterval);
}
}
}
static void comTrainMemQ(bool load) {
Console.WriteLine("Q is player..?");
byte temp3 = byte.Parse(Console.ReadLine());
MemoryQAgent mqagent = new MemoryQAgent(temp3);
if (load)
mqagent.Load("TrainingData");
Console.WriteLine("How many iterations?");
int temp1 = int.Parse(Console.ReadLine());
Console.WriteLine("Save how often?");
int temp2 = int.Parse(Console.ReadLine());
Console.WriteLine("Board size?");
int temp4 = int.Parse(Console.ReadLine());
Console.WriteLine("Which opponent?");
string command = Console.ReadLine();
TrainMemoryQ(temp4, temp3, mqagent, command, "log", "TrainingData", temp1, temp2);
}
public void TellOfMemoryTo(MemoryQAgent agent, bool fullepisode = false)
{
VirusMemory[] memories = LongTermMemory[random.Next(LongTermMemory.Count)].Memories;
VirusMemory memory = memories[0];
if (fullepisode)
{
foreach (VirusMemory m in memories)
{
agent.Learn(m);
}
}
else
{
foreach (VirusMemory m in memories)
{
if (Math.Abs(m.Reward) > Math.Abs(memory.Reward))
{
memory = m;
}
}
agent.Learn(memory);
}
}
static void TrainMemoryQ(int size, byte qnumber, MemoryQAgent agent, String opponent, String logname, String savename, int iterations, int saveinterval = 360) {
Virus virus = new Virus(2, size);
int wins = 0, wins2 = 0;
byte oppnumber = qnumber == 1 ? (byte)2 : (byte)1;
Agent opp = new BruteForceAgent(oppnumber);
StreamWriter writer = new StreamWriter(logname);
for (int i = 1; i <= iterations; i++) {
switch (opponent) {
case "brute":
break;
case "minimax4":
opp = new MinimaxAgent(4, oppnumber);
break;
case "minimax3":
opp = new MinimaxAgent(3, oppnumber);
break;
case "minimax2":
opp = new MinimaxAgent(2, oppnumber);
break;
default:
opp = new BruteForceAgent(oppnumber);
break;
}
int winner = RunGame(virus, qnumber == 1 ? (Agent)agent : opp, qnumber == 2 ? (Agent)agent : opp);
wins += winner == 1 ? 1 : 0;
wins2 += winner == 1 ? 1 : 0;
agent.ProcessShortTermMemory();
if (i % saveinterval == 0) {
agent.Save(savename);
Console.WriteLine("Iteration: " + i);
Console.WriteLine("Wins: " + wins);
wins = 0;
}
virus = new Virus(2, size);
}
for (int n = 0; n < 10; n++) {
MemoryQAgent ag = new MemoryQAgent();
for (int i = 0; i < 1000; i++) {
agent.TellOfMemoryTo(ag);
}
wins = 0;
opp = new BruteForceAgent(oppnumber);
for (int i = 1; i <= 10000; i++) {
switch (opponent) {
case "brute":
break;
case "minimax4":
opp = new MinimaxAgent(4, oppnumber);
break;
case "minimax3":
opp = new MinimaxAgent(3, oppnumber);
break;
case "minimax2":
opp = new MinimaxAgent(2, oppnumber);
break;
default:
opp = new BruteForceAgent(oppnumber);
break;
}
int winner = RunGame(virus, qnumber == 1 ? (Agent)ag : opp, qnumber == 2 ? (Agent)ag : opp);
wins += winner == 1 ? 1 : 0;
virus = new Virus(2, size);
}
Console.WriteLine("After 1000 memories");
Console.WriteLine("Wins: " + wins);
writer.Write(((double)wins) / 10000.0 + ";");
for (int i = 0; i < 9000; i++) {
agent.TellOfMemoryTo(ag);
}
wins = 0;
opp = new BruteForceAgent(oppnumber);
for (int i = 1; i <= 10000; i++) {
switch (opponent) {
case "brute":
break;
case "minimax4":
opp = new MinimaxAgent(4, oppnumber);
break;
case "minimax3":
opp = new MinimaxAgent(3, oppnumber);
break;
case "minimax2":
opp = new MinimaxAgent(2, oppnumber);
break;
default:
opp = new BruteForceAgent(oppnumber);
break;
}
int winner = RunGame(virus, qnumber == 1 ? (Agent)ag : opp, qnumber == 2 ? (Agent)ag : opp);
wins += winner == 1 ? 1 : 0;
virus = new Virus(2, size);
}
Console.WriteLine("After 10000 memories");
Console.WriteLine("Wins: " + wins);
writer.Write(((double)wins) / 10000.0 + ";");
for (int i = 0; i < 90000; i++) {
agent.TellOfMemoryTo(ag);
}
wins = 0;
opp = new BruteForceAgent(oppnumber);
for (int i = 1; i <= 10000; i++) {
switch (opponent) {
case "brute":
break;
case "minimax4":
opp = new MinimaxAgent(4, oppnumber);
break;
case "minimax3":
opp = new MinimaxAgent(3, oppnumber);
break;
case "minimax2":
opp = new MinimaxAgent(2, oppnumber);
break;
default:
opp = new BruteForceAgent(oppnumber);
break;
}
int winner = RunGame(virus, qnumber == 1 ? (Agent)ag : opp, qnumber == 2 ? (Agent)ag : opp);
wins += winner == 1 ? 1 : 0;
virus = new Virus(2, size);
}
Console.WriteLine("After 100000 memories");
Console.WriteLine("Wins: " + wins);
writer.WriteLine(((double)wins) / 10000.0);
}
writer.Close();
agent.SaveLongTermMemory("m");
}
public int TellOfMemoryToExt(MemoryQAgent agent, bool fullepisode = false) {
VirusMemory[] memories = LongTermMemory[random.Next(LongTermMemory.Count)].Memories;
VirusMemory memory = memories[0];
if (fullepisode) {
agent.LearnFromEpisode(memories, true);
//foreach (VirusMemory m in memories)
// agent.Learn(m);
return memories.Length;
}
else {
foreach (VirusMemory m in memories)
if (Math.Abs(m.Reward) > Math.Abs(memory.Reward))
memory = m;
agent.Learn(memory);
return 1;
}
}
public void TellOfMemoryTo(MemoryQAgent agent, bool fullepisode = false) {
VirusMemory[] memories = LongTermMemory[random.Next(LongTermMemory.Count)].Memories;
VirusMemory memory = memories[0];
if (fullepisode) {
foreach (VirusMemory m in memories)
agent.Learn(m);
}
else {
foreach (VirusMemory m in memories)
if (Math.Abs(m.Reward) > Math.Abs(memory.Reward))
memory = m;
agent.Learn(memory);
}
}
static void comTestMemQ2() {
//train a MemQ agent, and once every ten games have it teach an observer
//do this with ten agents
const int teachingagents = 10;
const int gamesbetweenteaching = 10;
const int learninggames = 10000;
int wins = 0;
int n = 0;
while (File.Exists("observer" + n + ".log")) n++;
StreamWriter writer = new StreamWriter("observer" + n + ".log");
writer.WriteLine("sources");
Virus virus;
MemoryQAgent observer = new MemoryQAgent(1);
Agent opponent = new BruteForceAgent(2);
MemoryQAgent[] agents = new MemoryQAgent[teachingagents];
for (int i = 0; i < teachingagents; i++) {
agents[i] = new MemoryQAgent(1);
wins = 0;
for (int j = 0; j < learninggames; j++) {
virus = new Virus(2, 5);
wins += RunGame(virus, agents[i], opponent) == 1 ? 1 : 0;
agents[i].ProcessShortTermMemory();
if (j % 1000 == 999) {
if (j > 9500) {
writer.WriteLine(wins);
Console.WriteLine("Wins: " + wins);
}
wins = 0;
}
}
}
writer.WriteLine("observer");
int totalmems = 0;
int memstoprocess = 0;
while (totalmems < 100000) {
//if (memstoprocess == 10000) memstoprocess = 100000;
//if (memstoprocess == 1000) memstoprocess = 10000;
//if (memstoprocess < 1000) memstoprocess = 1000;
memstoprocess += 1000;
while (totalmems < memstoprocess) {
for (int i = 0; i < teachingagents; i++) {
totalmems += agents[i].TellOfMemoryToExt(observer, true);
}
}
//writer.WriteLine("observer");
//for (int i = 0; i < 10; i++) {
wins = 0;
for (int j = 0; j < 10000; j++) {
virus = new Virus();
wins += RunGame(virus, observer, opponent) == 1 ? 1 : 0;
observer.ForgetShortTerm();
}
writer.WriteLine(wins);
Console.WriteLine("Observer wins after " + totalmems + "memories: " + wins);
//}
}
writer.Close();
}
public void StartGame(
Virus virus,
PerformedMoveCallback callback,
UpdatePiecesCallback piecesCallback,
EndCallback end,
string id,
params VirusPlayer[] players)
{
Random rand = new Random();
PerformedMove = callback;
UpdatePieces = piecesCallback;
End = end;
PlayerID = id;
this.virus = virus;
this.immediateAI = true;
this.MouseClick += MouseClickHandler1;
tileSize = 49;
this.Size = new Size(
virus.Size * tileSize + 17,
virus.Size * tileSize + 55);
int smallestSide = this.Size.Height < this.Size.Width ? this.Size.Height : this.Size.Width;
tileSize = smallestSide / virus.Size;
this.players.Add(new VirusPlayer("Player 0", "", Color.White));
this.players.AddRange(players);
while (this.players.Count < virus.Players + 1) {
this.players.Add(new VirusPlayer("BruteAI","AI",Color.FromArgb(rand.Next(256), rand.Next(256), rand.Next(256))));
}
//Save("Lalalafil");
agents = new Agent[this.players.Count];
int n = 1;
for (byte i = 1; i < this.players.Count; i++) {
String p = this.players[i].Name;
switch (p) {
case "AIQ":
agents[i] = new QAgent(i);
if (File.Exists("TrainingData.Q") && File.Exists("TrainingData.N")) {
((QAgent)agents[i]).Load("TrainingData");
((QAgent)agents[i]).TurnOffExploration();
((QAgent)agents[i]).TurnOffLearning();
}
//this.players[i].Name = "AI " + n;
n++;
break;
case "AIMQ":
agents[i] = new MemoryQAgent(i);
if (File.Exists("TrainingData.Q") && File.Exists("TrainingData.N")) {
((MemoryQAgent)agents[i]).Load("TrainingData");
((MemoryQAgent)agents[i]).TurnOffExploration();
}
//this.players[i].Name = "AI " + n;
n++;
break;
case "AIMinimax":
agents[i] = new MinimaxAgent(4,i);
//this.players[i].Name = "AI " + n;
n++;
break;
case "AIMiniMaxMix":
if (File.Exists("TrainingData.Q"))
agents[i] = new MiniMaxMixAgent("TrainingData", 2, i);
else
agents[i] = new BruteForceAgent(i);
//this.players[i].Name = "AI " + n;
n++;
break;
case "AIMixed":
agents[i] = new MixedAgent(0.5,false,i);
//this.players[i].Name = "AI " + n;
n++;
break;
case "AIBrute":
agents[i] = new BruteForceAgent(i);
//this.players[i].Name = "AI " + n;
n++;
break;
case "AIRandom":
agents[i] = new RandomAgent(i);
//this.players[i].Name = "AI " + n;
n++;
break;
case "AISimple":
agents[i] = new SimpleAgent(i);
//this.players[i].Name = "AI " + n;
n++;
break;
}
}
message = this.players[1].Name + "'s turn";
/*colors = new Color[virus.Players + 1];
colors[0] = Color.White;
colors[1] = Color.FromArgb(128, 160, 255);
colors[2] = Color.FromArgb(96, 255, 96);
if(virus.Players > 2)
colors[3] = Color.FromArgb(255, 96, 96);
if(virus.Players > 3)
colors[4] = Color.FromArgb(255, 255, 64);
for (int i = 5; i <= virus.Players; i++)
colors[i] = Color.FromArgb(rand.Next(256), rand.Next(256), rand.Next(256));*/
}
