private IEnumerator TakeStep()
{
//Get current car state
float[] current_state = car_camera.GetRays();
//Get action from current state
float[] q_values = particles[working_particle].Compute(current_state);
last_q_values = q_values;
action_index = SelectAction(q_values);
//Wait for action to complete
yield return(new WaitForSeconds(0.1f));
//Get next state
float[] next_state = car_camera.GetRays();
//Get max `a` of `q_target`
float[] q_target = network_target.Compute(next_state);
int max_q_target = 0;
for (int i = 1; i < max_q_target; i++)
{
if (q_target[max_q_target] < q_target[i])
{
max_q_target = i;
}
}
//Get rward for action
float velocity = car_body.gameObject.transform.InverseTransformDirection(car_body.velocity).z;
current_reward = velocity + reward_decay * q_target[max_q_target];
particles[working_particle].SetNetworkScore(current_reward);
//Reset car if stuck after 100 steps
if (car_body.velocity.magnitude < 0.3f && current_step - reset_step > 100)
{
reset_step = current_step;
car_body.transform.position = car_spawner.transform.position;
car_body.transform.rotation = car_spawner.transform.rotation;
car_body.velocity = Vector3.zero;
car_body.angularVelocity = Vector3.zero;
}
//After 300 steps go to next particle
if (current_step - particle_step > next_particle_wait)
{
working_particle++;
//reset reward
current_reward = 0;
particle_step = current_step;
}
//Do a pso update after all particles
if (working_particle == max_particles)
{
network_target.SetWeightsData(particle_swarm.GetBestWeights());
//PSO Update Step
particle_swarm.ComputeEpoch();
particle_swarm.UpdateWeights();
working_particle = 0;
}
current_step++;
if (!abort_learning)
{
StartCoroutine(TakeStep());
}
}
