private void button1_Click(object sender, EventArgs e)
{
double[] results = new double[8];
for (int num_agents = 1; num_agents <= 8; num_agents++)
{
// instantiate problem
EgoAlloGridWorldMulti world = new EgoAlloGridWorldMulti() { numAgents = num_agents };
List<Agent<int[], int[]>> agents = (List<Agent<int[], int[]>>)world.addAgent(typeof(OptimalPolicy<,>), typeof(LinearEgoAlloValue<,>), true, 1, true);
world.Load("C:\\Users\\Eric\\Google Drive\\Lethbridge Projects\\mapEgoAlloTrain.bmp");
for (int i = 0; i < 2000; i++)
{
world.stepAgent();
pictureBox1.Image = world.showState(100, 100);
pictureBox1.Refresh();
System.Threading.Thread.Sleep(10);
}
foreach (Agent<int[], int[]> a in agents)
((LinearEgoAlloValue<int[], int[]>)a._actionValue).ResetAllocentric(true);
world.Load("C:\\Users\\Eric\\Google Drive\\Lethbridge Projects\\mapEgoAlloTest.bmp");
PerformanceStats stats = new PerformanceStats();
int step = 0;
for (step = 0; step < 5000; step++)
{
stats = world.stepAgent();
pictureBox1.Image = world.showState(100, 100);
pictureBox1.Refresh();
System.Threading.Thread.Sleep(10);
if (stats.cumulativeReward > 0)
break;
}
results[num_agents - 1] = step;
}
foreach (double d in results)
textBox1.AppendText(d.ToString() + Environment.NewLine);
}
static void Main(string[] args)
{
//MultiResolutionRL.StateManagement.learnedStateTree tree = new MultiResolutionRL.StateManagement.learnedStateTree();
//IntArrayComparer comparer = new IntArrayComparer();
//System.IO.StreamReader rdr = new System.IO.StreamReader("C:\\Users\\Eric\\Google Drive\\Lethbridge Projects\\Fuzzy Place Field Test\\parents1.csv");
//rdr.ReadLine();
//string thisline;
//while ((thisline = rdr.ReadLine()) != null)
//{
// string[] elements = thisline.Split(',');
// int[] thisState = new int[2] { Convert.ToInt32(elements[0]), Convert.ToInt32(elements[1]) };
// int[] parent = tree.GetParentState(thisState, 3);
// if (comparer.Equals(parent, new int[2] { 3, 5 }))
// {
// int a = 0;
// }
// List<int[]> children = tree.GetLevel0Children(parent, 3);
//}
//// task-switch study
//int runs = 48;
//int goalCt = 10;
//List<double>[] stepsToGoal = new List<double>[runs];
//List<double>[] cumModelUse = new List<double>[runs];
//System.Diagnostics.Stopwatch sw = new System.Diagnostics.Stopwatch();
//sw.Start();
//ParallelOptions op = new ParallelOptions()
//{
// MaxDegreeOfParallelism = 8
//};
//Parallel.For(0, runs, op, (run) =>
//{
// cumModelUse[run] = new List<double>();
// // instantiate world
// World thisWorld = new GridWorld();
// // load 1st map
// thisWorld.Load("C:\\Users\\Eric\\Google Drive\\Lethbridge Projects\\map10.bmp");
// // add agent
// System.Threading.Thread.Sleep(run * 100); // staggered instantiation to avoid identical random number generators
// //thisWorld.addAgent(typeof(EGreedyPolicy<,>), typeof(MultiGridWorldModel<,>), 8);
// thisWorld.addAgent(typeof(EGreedyPolicy<,>), typeof(ModelBasedValue<,>));
// // run
// PerformanceStats stats = new PerformanceStats();
// while (stats.stepsToGoal.Count <= goalCt)
// {
// stats = thisWorld.stepAgent("");
// if (stats.stepsToGoal.Last() == 0)
// {
// cumModelUse[run].Add(stats.modelAccesses + stats.modelUpdates);
// Console.WriteLine("run " + run.ToString() + " goal count: " + stats.stepsToGoal.Count);
// }
// }
// // switch task
// thisWorld.Load("C:\\Users\\Eric\\Google Drive\\Lethbridge Projects\\map10e.bmp");
// // run again
// while (stats.stepsToGoal.Count <= goalCt * 2)
// {
// stats = thisWorld.stepAgent("");
// if (stats.stepsToGoal.Last() == 0)
// {
// cumModelUse[run].Add(stats.modelAccesses + stats.modelUpdates);
// Console.WriteLine("run " + run.ToString() + " goal count: " + stats.stepsToGoal.Count);
// }
// }
// stepsToGoal[run] = stats.stepsToGoal;
//});
//System.IO.StreamWriter writer = new System.IO.StreamWriter("C:\\Users\\Eric\\Google Drive\\Lethbridge Projects\\stepsToGoal.csv");
//for (int i = 0; i < stepsToGoal[0].Count; i++)
//{
// List<string> line = new List<string>();
// foreach (List<double> series in stepsToGoal)
// {
// line.Add(series[i].ToString());
// }
// writer.WriteLine(string.Join(",", line));
//}
//writer.Flush();
//writer.Close();
//writer = new System.IO.StreamWriter("C:\\Users\\Eric\\Google Drive\\Lethbridge Projects\\modelUse.csv");
//for (int i = 0; i < cumModelUse[0].Count; i++)
//{
// List<string> line = new List<string>();
// foreach (List<double> series in cumModelUse)
// {
// line.Add(series[i].ToString());
// }
// writer.WriteLine(string.Join(",", line));
//}
//writer.Flush();
//writer.Close();
// stochastic reward study
int runs = 48;
int goalCt = 100;
List<double>[] stepsToGoal = new List<double>[runs];
List<double>[] cumModelUse = new List<double>[runs];
System.Diagnostics.Stopwatch sw = new System.Diagnostics.Stopwatch();
sw.Start();
ParallelOptions op = new ParallelOptions()
{
MaxDegreeOfParallelism = 8
};
Parallel.For(0, runs, op, (run) =>
{
cumModelUse[run] = new List<double>();
// instantiate world
World thisWorld = new StochasticRewardGridWorld();
// load map
thisWorld.Load("C:\\Users\\Eric\\Google Drive\\Lethbridge Projects\\map4choiceB.bmp");
// add agent
System.Threading.Thread.Sleep(run * 100); // staggered instantiation to avoid identical random number generators
thisWorld.addAgent(typeof(EGreedyPolicy<,>), typeof(ModelBasedValue<,>));
//thisWorld.addAgent(typeof(EGreedyPolicy<,>), typeof(ContextSwitchValue<,>), 8, 100);
// run
PerformanceStats stats = new PerformanceStats();
while (stats.stepsToGoal.Count <= goalCt)
{
stats = thisWorld.stepAgent("");
if (stats.stepsToGoal.Last() == 0)
{
cumModelUse[run].Add(stats.modelAccesses + stats.modelUpdates);
Console.WriteLine("run " + run.ToString() + " goal count: " + stats.stepsToGoal.Count);
}
}
stepsToGoal[run] = stats.stepsToGoal;
});
System.IO.StreamWriter writer = new System.IO.StreamWriter("C:\\Users\\Eric\\Google Drive\\Lethbridge Projects\\stepsToGoalStochasticMBRL.csv");
for (int i = 0; i < stepsToGoal[0].Count; i++)
{
List<string> line = new List<string>();
foreach (List<double> series in stepsToGoal)
{
line.Add(series[i].ToString());
}
writer.WriteLine(string.Join(",", line));
}
writer.Flush();
writer.Close();
writer = new System.IO.StreamWriter("C:\\Users\\Eric\\Google Drive\\Lethbridge Projects\\modelUseStochasticMBRL.csv");
for (int i = 0; i < cumModelUse[0].Count; i++)
{
List<string> line = new List<string>();
foreach (List<double> series in cumModelUse)
{
line.Add(series[i].ToString());
}
writer.WriteLine(string.Join(",", line));
}
writer.Flush();
writer.Close();
//// Lesion study
//int runs = 8;
//int goalCt = 25;
//List<double>[] results = new List<double>[runs];
//System.Diagnostics.Stopwatch sw = new System.Diagnostics.Stopwatch();
//sw.Start();
//ParallelOptions op = new ParallelOptions()
//{
// MaxDegreeOfParallelism = 8
//};
//Parallel.For(0, runs, op, (run) =>
////for (int run = 0; run < runs; run++)
//{
// // instantiate world
// World thisWorld = new GridWorld();
// // load map
// thisWorld.Load("C:\\Users\\Eric\\Google Drive\\Lethbridge Projects\\map3LargeMod.bmp");
// // load agent
// System.Threading.Thread.Sleep(run * 100); // staggered instantiation to avoid identical random number generators
// //thisWorld.addAgent(typeof(SoftmaxPolicy<,>), typeof(MultiGridWorldModel<,>), 8, 4);
// thisWorld.addAgent(typeof(EGreedyPolicy<,>), typeof(MultiResValue<,>), 1, 0);
// // run
// PerformanceStats stats = new PerformanceStats();
// while (stats.stepsToGoal.Count <= goalCt)
// {
// stats = thisWorld.stepAgent("");
// }
// results[run] = stats.stepsToGoal;
//});
//sw.Stop();
//Console.WriteLine(sw.Elapsed.TotalSeconds.ToString());
//System.IO.StreamWriter writer = new System.IO.StreamWriter("C:\\Users\\Eric\\Google Drive\\Lethbridge Projects\\data.csv");
//for (int i = 0; i < goalCt; i++)
//{
// List<string> line = new List<string>();
// foreach (List<double> series in results)
// {
// line.Add(series[i].ToString());
// }
// writer.WriteLine(string.Join(",", line));
//}
//writer.Flush();
//writer.Close();
//// Post-training Lesion study
//int runs = 7;
//int goalCt = 2;
//List<double>[] results = new List<double>[runs];
//System.Diagnostics.Stopwatch sw = new System.Diagnostics.Stopwatch();
//sw.Start();
//Parallel.For(0, runs, (run) =>
////for (int run = 0; run < runs; run++)
//{
// // instantiate world
// World thisWorld = new GridWorld();
// // load map
// thisWorld.Load("C:\\Users\\Eric\\Google Drive\\Lethbridge Projects\\map3.bmp");
// // load agent
// System.Threading.Thread.Sleep(run * 100); // staggered instantiation to avoid identical random number generators
// //thisWorld.addAgent(typeof(SoftMaxPolicy<,>), typeof(MultiGridWorldModel<,>), 8, 4);
// Agent<int[], int[]> agent = (Agent<int[], int[]>)thisWorld.addAgent(typeof(EGreedyPolicy<,>), typeof(MultiGridWorldModel<,>), 8, 4);
// // run
// PerformanceStats stats = new PerformanceStats();
// while (stats.stepsToGoal.Count <= goalCt)
// {
// stats = thisWorld.stepAgent("");
// }
// // change environment
// thisWorld.Load("C:\\Users\\Eric\\Google Drive\\Lethbridge Projects\\map3b.bmp");
// // lesion vH
// MultiGridWorldModel<int[], int[]> model = (MultiGridWorldModel<int[], int[]>)agent._actionValue;
// model.LesionVH(1);
// // run
// while (stats.stepsToGoal.Count <= goalCt * 2)
// {
// stats = thisWorld.stepAgent("");
// }
// results[run] = stats.stepsToGoal;
//});
//sw.Stop();
//Console.WriteLine(sw.Elapsed.TotalSeconds.ToString());
//System.IO.StreamWriter writer = new System.IO.StreamWriter("C:\\Users\\Eric\\Google Drive\\Lethbridge Projects\\data.csv");
//for (int i = 0; i < goalCt; i++)
//{
// List<string> line = new List<string>();
// foreach (List<double> series in results)
// {
// line.Add(series[i].ToString());
// }
// writer.WriteLine(string.Join(",", line));
//}
//writer.Flush();
//writer.Close();
}
static void Main(string[] args)
{
int wS = 48; //WorldSize p[0]
bool RL = true; //RLMethod p[1]; 'F' for QL, 'T' For MB
double a = 0.1; //alpha p[2];
double g = 0.8; //Gamma p[3];
int tO = wS; //timeOut p[4];
double mR = 1; //Manager Rewards p[5];
Policy<int[], int[]> cP = new EGreedyPolicy<int[], int[]>(); //chosen Policy p[6]
// task-switch test
int runs = 48;
int goalCt = 10;
List<double>[] stepsToGoal = new List<double>[runs];
List<double>[] cumModelUse = new List<double>[runs];
System.Diagnostics.Stopwatch sw = new System.Diagnostics.Stopwatch();
sw.Start();
ParallelOptions op = new ParallelOptions()
{
MaxDegreeOfParallelism = 8
};
//for (int run = 0; run < runs; run++)
Parallel.For(0, runs, op, run =>
{
cumModelUse[run] = new List<double>();
// instantiate world
World thisWorld = new GridWorld();
// load 1st map
thisWorld.Load("C:\\Users\\Eric\\Google Drive\\Lethbridge Projects\\map10.bmp");
// add agent
System.Threading.Thread.Sleep(run * 100); // staggered instantiation to avoid identical random number generators
//thisWorld.addAgent(typeof(EGreedyPolicy<,>), typeof(MultiGridWorldModel<,>), 8);
// thisWorld.addAgent(typeof(EGreedyPolicy<,>), typeof(Boss<,>), wS, RL, a, g, tO, mR, cP);
// run
PerformanceStats stats = new PerformanceStats();
while (stats.stepsToGoal.Count <= goalCt)
{
stats = thisWorld.stepAgent("");
if (stats.stepsToGoal.Last() == 0)
{
cumModelUse[run].Add(stats.modelAccesses + stats.modelUpdates);
Console.WriteLine("run " + run.ToString() + " goal count: " + stats.stepsToGoal.Count);
}
}
// switch task
thisWorld.Load("C:\\Users\\Eric\\Google Drive\\Lethbridge Projects\\map10a.bmp");
// run again
while (stats.stepsToGoal.Count <= goalCt * 2)
{
stats = thisWorld.stepAgent("");
if (stats.stepsToGoal.Last() == 0)
{
cumModelUse[run].Add(stats.modelAccesses + stats.modelUpdates);
Console.WriteLine("run " + run.ToString() + " goal count: " + stats.stepsToGoal.Count);
}
}
stepsToGoal[run] = stats.stepsToGoal;
});
System.IO.StreamWriter writer = new System.IO.StreamWriter("C:\\Users\\Eric\\Google Drive\\Lethbridge Projects\\stepsToGoal.csv");
for (int i = 0; i < stepsToGoal[0].Count; i++)
{
List<string> line = new List<string>();
foreach (List<double> series in stepsToGoal)
{
line.Add(series[i].ToString());
}
writer.WriteLine(string.Join(",", line));
}
writer.Flush();
writer.Close();
writer = new System.IO.StreamWriter("C:\\Users\\Eric\\Google Drive\\Lethbridge Projects\\modelUse.csv");
for (int i = 0; i < cumModelUse[0].Count; i++)
{
List<string> line = new List<string>();
foreach (List<double> series in cumModelUse)
{
line.Add(series[i].ToString());
}
writer.WriteLine(string.Join(",", line));
}
writer.Flush();
writer.Close();
}
private void button1_Click(object sender, EventArgs e)
{
Console.WriteLine("******");
System.Diagnostics.Stopwatch sw = new System.Diagnostics.Stopwatch();
sw.Start();
PerformanceStats stats = new PerformanceStats();
for (int i=0; i<(int)stepsUpDown.Value; i++)
{
stats = world.stepAgent(actionTextBox.Text);
//trajWriter.WriteLine(string.Join(",", ((Agent<int[], int[]>)agent).state));
label1.Text = i.ToString();
label1.Refresh();
chart1.Series.Last().Points.AddY(stats.cumulativeReward);
chart2.Series.Last().Points.AddY(stats.modelAccesses + stats.modelUpdates);
writer.WriteLine("Reward: " + stats.cumulativeReward);
if (displayCheckBox.Checked)
{
pictureBox1.Image = world.showState(pictureBox1.Width, pictureBox1.Height, true);
pictureBox1.Refresh();
Thread.Sleep(20);
if (saveImages)
{
pictureBox1.Image.Save(saveFolder + numSavedImages.ToString() + ".bmp");
numSavedImages++;
}
}
}
sw.Stop();
chart3.Series.Last().Points.Clear();
foreach (double d in stats.stepsToGoal)
{
if (d <= 0)
break;
chart3.Series.Last().Points.Add(d);
}
label1.Text = Math.Round(sw.Elapsed.TotalSeconds,1) + "s";
pictureBox1.Image = world.showState(pictureBox1.Width, pictureBox1.Height, true);
//System.IO.StreamReader r = new System.IO.StreamReader("log.txt");
//string text = r.ReadLine();
//if (text==null || (text.IndexOf("null")!=-1))
// pictureBox1.Image = world.showState(pictureBox1.Width, pictureBox1.Height);
//else
//{
// int start = text.IndexOf("Level ") + 6;
// string goalLevelString = text.Substring(start, 1);
// int goalLevel = Convert.ToInt32(goalLevelString);
// start = text.IndexOf("at ") + 3;
// string[] goalString = text.Substring(start).Split(',');
// int[] goal = new int[2];
// goal[0] = Convert.ToInt32(goalString[0]);
// goal[1] = Convert.ToInt32(goalString[1]);
// pictureBox1.Image = world.showState(pictureBox1.Width, pictureBox1.Height, true);
//}
//r.Close();
// chart cumulative reward
//chart1.Series.Last().Points.Clear();
//for (int i = 0; i < world.agent.cumulativeReward.Count; i++)
//{
// chart1.Series.Last().Points.AddXY(i, world.agent.cumulativeReward[i]);
//}
writer.Flush();
//writer.Close();
}
private void button2_Click(object sender, EventArgs e)
{
int runs = 64;
int trials = 50;
int stepCap = 200;
double[][] results = new double[trials * 2][];
for (int i = 0; i < (trials * 2); i++)
results[i] = new double[runs];
int scale = 2;
Bitmap map = new Bitmap("C:\\Users\\Eric\\Google Drive\\Lethbridge Projects\\waterMazeSim\\Open.bmp");
Bitmap resized = new Bitmap(map.Width * 3, map.Height * 3);
using (Graphics g = Graphics.FromImage(resized))
{
g.InterpolationMode = System.Drawing.Drawing2D.InterpolationMode.NearestNeighbor;
g.PixelOffsetMode = System.Drawing.Drawing2D.PixelOffsetMode.Half;
g.DrawImage(map, 0, 0, map.Width * scale, map.Height * scale);
}
resized.Save("temp.bmp");
map = new Bitmap("C:\\Users\\Eric\\Google Drive\\Lethbridge Projects\\waterMazeSim\\Third.bmp");
resized = new Bitmap(map.Width * 3, map.Height * 3);
using (Graphics g = Graphics.FromImage(resized))
{
g.InterpolationMode = System.Drawing.Drawing2D.InterpolationMode.NearestNeighbor;
g.PixelOffsetMode = System.Drawing.Drawing2D.PixelOffsetMode.Half;
g.DrawImage(map, 0, 0, map.Width * scale, map.Height * scale);
}
resized.Save("temp2.bmp");
Parallel.For(0, runs, run => // for (int run = 0; run < runs; run++)
{
GridWorld world = new GridWorld();
world.Load("temp.bmp");
world.addAgent(typeof(EGreedyPolicy<,>), typeof(MultiResValue<,>), 1, 0);
PerformanceStats stats = new PerformanceStats();
while (stats.stepsToGoal.Count < (trials + 1))
{
stats = world.stepAgent();
//pictureBox1.Image = world.showState(300, 300);
//pictureBox1.Refresh();
//System.Threading.Thread.Sleep(1);
if (stats.stepsToGoal.Last() >= stepCap)
{
stats.TallyStepsToGoal(true);
world.Load("temp.bmp");
}
}
world.Load("temp2.bmp");
while (stats.stepsToGoal.Count < (trials * 2 + 1))
{
stats = world.stepAgent();
//pictureBox1.Image = world.showState(300, 300);
//pictureBox1.Refresh();
//System.Threading.Thread.Sleep(1);
if (stats.stepsToGoal.Last() >= stepCap)
{
stats.TallyStepsToGoal(true);
world.Load("temp2.bmp");
}
}
for (int i = 0; i < trials * 2; i++)
{
results[i][run] = stats.stepsToGoal[i];
}
});
for (int i = 0; i < trials * 2; i++)
{
textBox1.AppendText(results[i].Average() + Environment.NewLine);
}
}
static void Main(string[] args)
{
string stepsToGoalFilename = "C:\\Users\\Eric\\Desktop\\Maps\\stepsToGoal.csv";
string modelUseFilename = "C:\\Users\\Eric\\Desktop\\Maps\\modelUse.csv";
string cumRewardFilename = "C:\\Users\\Eric\\Desktop\\Maps\\cumReward.csv";
string mapsDirectory = "C:\\Users\\Eric\\Desktop\\Maps\\";
int runs = 96;
int goalCt = 10;
List<double>[] stepsToGoal = new List<double>[runs];
List<double>[] cumModelUse = new List<double>[runs];
List<double>[] cumReward = new List<double>[runs];
string[] mapNames = Directory.GetFiles(mapsDirectory, "*.bmp");
List<string> maps = new List<string>();
maps.AddRange(mapNames); //maps.AddRange(mapNames);
System.Diagnostics.Stopwatch sw = new System.Diagnostics.Stopwatch();
sw.Start();
ParallelOptions op = new ParallelOptions()
{
MaxDegreeOfParallelism = 8
};
//for(int run=0; run< runs; run++)
Parallel.For(0, runs, op, (run) =>
{
cumModelUse[run] = new List<double>();
cumReward[run] = new List<double>();
// instantiate world
World thisWorld = new GridWorld();
// add agent
System.Threading.Thread.Sleep(run * 101); // staggered instantiation to avoid identical random number generators
thisWorld.addAgent(typeof(EGreedyPolicy<,>), typeof(ContextSwitchValue<,>), 8, 100); // this line for context-switch + adaptation
//thisWorld.addAgent(typeof(EGreedyPolicy<,>), typeof(ContextSwitchValue<,>), 1, 100); // this line for context-switch only
//thisWorld.addAgent(typeof(EGreedyPolicy<,>), typeof(ModelBasedValue<,>)); // this line for standard MBRL
PerformanceStats stats = new PerformanceStats();
for (int mapNumber = 0; mapNumber<maps.Count; mapNumber++)
{
// load map
thisWorld.Load(maps[mapNumber]);
// go
while (stats.stepsToGoal.Count < goalCt * (mapNumber+1))
{
stats = thisWorld.stepAgent("");
if (stats.stepsToGoal.Last() == 0)
{
cumModelUse[run].Add(stats.modelAccesses + stats.modelUpdates);
cumReward[run].Add(stats.cumulativeReward);
Console.WriteLine("run " + run.ToString() + " goal count: " + (stats.stepsToGoal.Count-1) + " steps: " + stats.stepsToGoal[mapNumber]);
}
}
stepsToGoal[run] = stats.stepsToGoal;
}
});
//}
saveToCSV(stepsToGoalFilename, stepsToGoal);
saveToCSV(modelUseFilename, cumModelUse);
saveToCSV(cumRewardFilename, cumReward);
}