public void Train()
{
GenerateRVector();
LoadMMatrices("mmatrices.dat");
Vector[] Q = new Vector[CarNavigationQStore.LENACTION];
for (int i = 0; i < CarNavigationQStore.LENACTION; ++i)
{
Q[i] = new Vector(statenum);
}
for (int i = 0; i < CarNavigationQStore.LENACTION; ++i)
{
for (int j = 0; j < CarNavigationQStore.LENXY; ++j)
{
for (int k = 0; k < CarNavigationQStore.LENXY; ++k)
{
for (int l = 0; l < CarNavigationQStore.LENANG; ++l)
{
Q[i].Elements[l * CarNavigationQStore.LENXY * CarNavigationQStore.LENXY + k * CarNavigationQStore.LENXY + j] = qstore.value[i, j, k, l];
}
}
}
}
Vector[] p_ = new Vector[CarNavigationQStore.LENACTION];
float max_ = float.MinValue;
for (int i = 0; i < CarNavigationQStore.LENACTION; ++i)
{
p_[i] = new Vector(Q[i]);
for (int j = 0; j < statenum; ++j)
{
if (max_ < p_[i].Elements[j]) max_ = (float)p_[i].Elements[j];
}
}
for (int i = 0; i < CarNavigationQStore.LENACTION; ++i)
{
p_[i].Exp(-max_);
}
Vector sum = new Vector(statenum);
for (int i = 0; i < CarNavigationQStore.LENACTION; ++i)
{
sum += p_[i];
}
SparseMatrix[] Ma_ = new SparseMatrix[CarNavigationQStore.LENACTION];
for (int i = 0; i < CarNavigationQStore.LENACTION; ++i)
{
p_[i].Div(sum);
Ma_[i] = new SparseMatrix(Ma[i]);
Ma_[i].Multiply(p_[i]);
}
//M matrix kiszamitasa
SparseMatrix M = new SparseMatrix(statenum);
for (int i = 0; i < CarNavigationQStore.LENACTION; ++i)
{
M.Add(Ma_[i]);
}
M.Multiply(0.9999f);
SparseMatrix IM = SparseMatrix.Identity(statenum) - M;
//IM.WriteToFile("IM.txt");
Vector utility = IM.SolveLinearEquation2(R);
for (int i = 0; i < CarNavigationQStore.LENACTION; ++i)
{
Q[i] = Ma[i].MatrixMultiplyRight(utility);
}
Vector QMax = new Vector(statenum);
for (int j = 0; j < statenum; ++j)
{
float max = float.MinValue;
int best = 0;
for (int i = 0; i < CarNavigationQStore.LENACTION; ++i)
{
if (Q[i].Elements[j] > max)
{
max = (float)Q[i].Elements[j];
best = i;
}
}
QMax.Elements[j] = best;
}
for (int i = 0; i < CarNavigationQStore.LENACTION; ++i)
{
for (int j = 0; j < CarNavigationQStore.LENXY; ++j)
{
for (int k = 0; k < CarNavigationQStore.LENXY; ++k)
{
for (int l = 0; l < CarNavigationQStore.LENANG; ++l)
{
qstore.value[i, j, k, l] = (float)Q[i].Elements[l * CarNavigationQStore.LENXY * CarNavigationQStore.LENXY + k * CarNavigationQStore.LENXY + j];
}
}
}
}
utility.WriteToFile("u.txt");
qstore.Save("qstore.dat");
//Q[0].WriteToFile("q1.txt");
//Q[1].WriteToFile("q2.txt");
//Q[2].WriteToFile("q3.txt");
//Q[3].WriteToFile("q4.txt");
//Q[4].WriteToFile("q5.txt");
//Q[5].WriteToFile("q6.txt");
//Q[6].WriteToFile("q7.txt");
//Q[7].WriteToFile("q8.txt");
//Q[8].WriteToFile("q9.txt");
//Q[9].WriteToFile("q10.txt");
//Q[10].WriteToFile("q11.txt");
}
public static SparseMatrix operator -(SparseMatrix a, SparseMatrix b)
{
if (a.size != b.size)
{
return null;
}
SparseMatrix ret = new SparseMatrix(a);
ret.Add(b, -1);
return ret;
}