protected override void OnPaint(PaintEventArgs pe)
{
base.OnPaint(pe);
Graphics g = pe.Graphics;
g.Clear(Color.White);
if (QStore != null)
{
CarNavigationEnvironment testenv = new CarNavigationEnvironment();
for (int xx = 0; xx < CarNavigationQStore.LENXY; xx+=2)
{
for (int yy = 0; yy < CarNavigationQStore.LENXY; yy+=2)
{
for (int ang = 0; ang < CarNavigationQStore.LENANG; ang+=2)
{
CarNavigationState state;
CarNavigationQStore.GetState(xx, yy, ang, out state);
testenv.x = state.x;
testenv.y = state.y;
testenv.alpha = state.alpha;
double ax = testenv.x;
double ay = testenv.y;
g.DrawEllipse(new Pen(Color.Red), Width / 2 + (float)ax * scale, Height / 2 + (float)ay * scale, 3, 3);
int len = 0;
do
{
ReinforcementLearningAction action;
float utility;
QStore.GetBestActionAndUtilityForState(testenv.State(), out action, out utility);
testenv.Step(((CarNavigationAction)action).ang);
double bx = testenv.x;
double by = testenv.y;
g.DrawLine(new Pen(Color.FromArgb(32, (int)((Math.Cos(testenv.alpha) + 1) * 127), (int)((Math.Sin(testenv.alpha) * Math.Cos(testenv.alpha) + 1) * 127), (int)((Math.Sin(testenv.alpha) + 1) * 127))), Width / 2 + (float)ax * scale, Height / 2 + (float)ay * scale, Width / 2 + (float)bx * scale, Height / 2 + (float)by * scale);
ax = bx;
ay = by;
++len;
}
while (!((Math.Abs(ax) < 0.25) && (Math.Abs(ay) < 0.25) && (Math.Cos(testenv.alpha) < -0.9)) && (len < 100));
}
}
}
}
}
private void GenerateRVector()
{
CarNavigationEnvironment testws = new CarNavigationEnvironment();
R = new Vector(statenum);
float sum = 0;
for (int ii = 0; ii < statenum; ++ii)
{
int l = ii / (CarNavigationQStore.LENXY * CarNavigationQStore.LENXY);
int iil = ii - l * (CarNavigationQStore.LENXY * CarNavigationQStore.LENXY);
int k = iil / CarNavigationQStore.LENXY;
int j = iil % CarNavigationQStore.LENXY;
CarNavigationState state;
CarNavigationQStore.GetState(j, k, l, out state);
testws.x = state.x;
testws.y = state.y;
testws.alpha = state.alpha;
R.Elements[ii] = testws.Reward();
sum += (float)R.Elements[ii];
}
Console.Out.Write(sum);
}
public void GenerateMMatrices()
{
CarNavigationEnvironment testws = new CarNavigationEnvironment();
Ma = new SparseMatrix[CarNavigationQStore.LENACTION];
for (int i = 0; i < CarNavigationQStore.LENACTION; ++i)
{
Ma[i] = new SparseMatrix(statenum);
}
for (int ii = 0; ii < statenum; ++ii)
{
int l = ii / (CarNavigationQStore.LENXY * CarNavigationQStore.LENXY);
int iil = ii - l * (CarNavigationQStore.LENXY * CarNavigationQStore.LENXY);
int k = iil / CarNavigationQStore.LENXY;
int j = iil % CarNavigationQStore.LENXY;
CarNavigationState state;
CarNavigationQStore.GetState(j, k, l, out state);
for (int i = 0; i < CarNavigationQStore.LENACTION; ++i)
{
CarNavigationAction action;
CarNavigationQStore.GetAction(i, out action);
for (int i1 = 0; i1 < 100; ++i1)
{
testws.x = state.x + r.NextDouble() * (CarNavigationQStore.MAXXY - CarNavigationQStore.MINXY) / CarNavigationQStore.LENXY;
testws.y = state.y + r.NextDouble() * (CarNavigationQStore.MAXXY - CarNavigationQStore.MINXY) / CarNavigationQStore.LENXY;
testws.alpha = state.alpha + r.NextDouble() * (2 * Math.PI) / CarNavigationQStore.LENANG;
testws.Step(action.ang);
int j2, k2, l2;
CarNavigationState state2 = new CarNavigationState(testws.x, testws.y, testws.alpha);
CarNavigationQStore.GetStateIndices(state2, out j2, out k2, out l2);
int ii2 = l2 * CarNavigationQStore.LENXY * CarNavigationQStore.LENXY + k2 * CarNavigationQStore.LENXY + j2;
if (ii2 > statenum - 1) throw new Exception();
Ma[i][ii, ii2] += 0.01f;
}
}
}
SaveMMatrices("mmatrices.dat");
}