public static void vissimRunFirstInterval()
{
for (int i = 0; i < 180 * SimRes; i++)
{
VissimTools.RunSingleStep();
}
}
//Provide initial state(flowrate) to agent
public static Dictionary <string, double> vissimState(int count, int run_times, int[] action) //SimPeriod * SimRes
{
Set_AllDesireSpeed(action);
double flowrate = 0.0;
double density = 0.0;
for (int i = 0; i < run_times; i++)
{
VissimTools.RunSingleStep();
}
// int interval_num = (int)(run_times / (SimRes * DataCollectionInterval) * count + 1);
int datapoint1_vehs = VissimTools.Get_CurrentDataCollectionResult_Vehs(1);
flowrate = VissimTools.Get_FlowRate(datapoint1_vehs, DataCollectionInterval);
int num_lanes = VissimTools.Get_NumLaneByVehTravelTm(2);
int travelTm_vhs = Get_CurrentVehicleTravelTime_Vehs(2);
double distance = Get_CurrentVehicleTravelTime_DistTrav(2);
int time = Get_CurrentVehicleTravelTime_TravTm(2);
density = Get_Density(travelTm_vhs, DataCollectionInterval, time, distance, num_lanes);
Dictionary <string, double> result = new Dictionary <string, double>();
result.Add("flowrate", flowrate);
result.Add("density", density);
return(result);
}
static public double Get_Link_Desity(int lnk)
{
double length = System.Convert.ToDouble(vissim.Net.Links.get_ItemByKey(lnk).get_AttValue("Length2D"));
int num_lanes = System.Convert.ToInt32(vissim.Net.Links.get_ItemByKey(lnk).get_AttValue("NumLanes"));
object[,] temp = (object[, ])VissimTools.GetLinkVehiclesbyNumber(lnk);
int num_vehs = temp.Length / 2;
double density = num_vehs / (num_lanes * length / 1600); // veh/mi/ln
return(Math.Round(density, 0));
}
public static double vissimReward(int run_times, int[] actions) //SimPeriod * SimRes
{
Set_AllDesireSpeed(actions);
double Reward = 0;
for (int i = 0; i < run_times; i++)
{
VissimTools.RunSingleStep();
}
int datapoint4_vehs = VissimTools.Get_CurrentDataCollectionResult_Vehs(4);
Reward = VissimTools.Get_FlowRate(datapoint4_vehs, DataCollectionInterval);
// int density = VissimTools.Get_Density(int num_vehs, int timeinterval, double speed, double distance, int num_lane)
return(Reward);
}
public static bool train(int epoch, int batch)
{
int count = 0;
int action_runtime = 180 * 5;
int state_runtime = 180 * 5;
string outputPath = "./outputs";
List <string> errors = new List <string>();
List <string> records = new List <string>();
//Initialize all the instances
ShAgent agent = new ShAgent();
VissimTools.InitVissimTools();
VissimTools.vissimRunFirstInterval();
for (int e = 0; e < epoch; e++)
{
var watch = System.Diagnostics.Stopwatch.StartNew();
int batch_count = 0;
double batch_diff = 0;
for (int b = 0; b < batch; b++)
{
//get initial state from vissim to agent
double raw_state = VissimTools.vissimState(count, state_runtime); // TBD:
// map to discrete
if (raw_state < agent.get_min_state() || raw_state > agent.get_max_state())
{
count++;
continue;
}
batch_count++;
int state = (int)((raw_state - agent.get_min_state()) / agent.state_interval);
// temp store of current q table for convergence check
double[,] current_q_table = agent.q_table.Clone() as double[, ];
// get action
List <int> action = agent.get_action(state);
// get reward
int limSpd = (int)(agent.get_min_action() + action[0] * agent.action_interval);
double reward = VissimTools.vissimReward(action_runtime, limSpd);
// update q table
agent.update_q_table(state, action[0], reward);
/*
* Application app = new Application { Visible = false };
* Workbook xBook = app.Workbooks._Open(@"D:\School11111111111111111111111111111\Coop2019\Summer\Capacity\Data\data.xlsx");
* Worksheet xSheet = (Worksheet)xBook.Sheets[1];
* Range c1 = (Range)app.Cells[count * 13 + 1, 1];
* Range c2 = (Range)app.Cells[count * 13 + 1 + 13 - 1, 10];
* Range range = app.get_Range(c1, c2);
* range.Value = agent.q_table;
* xBook.Save();
* xSheet = null;
* xBook.Close(0); //xlBook.Close(true); //Workbook.close(SaveChanges, filename, routeworkbook )
* app.Quit();
*/
double diff = 0;
count++;
diff += Math.Abs(agent.q_table[state, action[0]] - current_q_table[state, action[0]]);
/*
* for (int m = 0; m < agent.get_state_size(); m++)
* {
* for (int n = 0; n < agent.get_action_size(); n++)
* {
* diff += Math.Abs(agent.q_table[m, n] - current_q_table[m, n]);
* }
* }
*/
System.Console.WriteLine("Run #:" + count + " ----------- " + diff + " (" + state + " : " + action[0] + " : " + action[1] + ")");
records.Add(count + "," + diff + "," + state + "," + action[0] + "," + action[1] + "," + reward);
batch_diff += diff;
GC.Collect();
}
// TBD: check if this is a good check
double error = batch_diff / batch_count;
errors.Add(error + "");
System.Console.WriteLine("Error: ================== >" + error);
if ((batch_diff / batch_count) < agent.convergence)
{
summary(records, outputPath, "final_outputs.csv");
summary(errors, outputPath, "final_error_outputs.csv");
save_q(agent.q_table, "final");
return(true);
}
watch.Stop();
System.Console.WriteLine("Epoch " + e + " execution time: ========== >" + watch.ElapsedMilliseconds);
summary(records, outputPath, "temp_outputs_epoch_" + e + ".csv");
summary(errors, outputPath, "temp_error_outputs_epoch_" + e + ".csv");
save_q(agent.q_table, e + "");
GC.Collect();
}
return(true);
}
