/// <summary>
/// Construct a ADP instance
/// </summary>
/// <param name="grid">The grid instance which trying to learn</param>
/// <param name="A">The list of valid actions</param>
/// <param name="gamma">The discount factor</param>
/// <param name="alpha">The learning rate</param>
public ADP(Grid grid, List<Action> A, float gamma, float alpha)
: base(grid, A, gamma, alpha)
{
this.NSA = new Hashtable();
this.NSAS = new Hashtable();
this.UTable = new Hashtable();
this.TTable = new Hashtable();
this.InitialState = grid.AgentPoint;
}
/// <summary>
/// Init the helper
/// </summary>
/// <param name="grid">The grid to bind with the helper</param>
public GridHelper(Grid grid)
{
this.Bind(grid);
this.propExaminer = new Random(System.Environment.TickCount);
this.RefreshTimer = new System.Timers.Timer(400);
this.RefreshTimer.Elapsed += new System.Timers.ElapsedEventHandler((sender, e) =>
{
lock (this.propExaminer)
this.propExaminer = new Random(System.Environment.TickCount + e.SignalTime.Millisecond);
});
this.RefreshTimer.Start();
}
/// <summary>
/// Construct a ADP instance
/// </summary>
/// <param name="grid">The grid instance which trying to learn</param>
/// <param name="A">The list of valid actions</param>
/// <param name="gamma">The discount factor</param>
/// <param name="alpha">The learning rate</param>
/// <param name="QTable">The initial Q-table(Can be also `null`)</param>
public ADP(Grid grid, List<Action> A, float gamma, float alpha, Hashtable QSA) : this(grid, A, gamma, alpha) { if (QSA != null)this.QTable = QSA; }
/// <summary>
/// Bind a grid to the helper
/// </summary>
/// <param name="grid">The target grid</param>
public void Bind(Grid grid) { this.BoundedGrid = grid; }
/// <summary>
/// Construct a SARSA-learner instance
/// </summary>
/// <param name="grid">The grid instance which trying to learn</param>
/// <param name="A">The list of valid actions</param>
/// <param name="gamma">The discount factor</param>
/// <param name="alpha">The learning rate</param>
/// <param name="QTable">The initial Q-table(Can be also `null`)</param>
public SarsaLearning(Grid grid, List<Action> A, float gamma, float alpha, Hashtable QSA) : base(grid, A, gamma, alpha, QSA) { }
/// <summary>
/// Construct a SARSA-learner instance
/// </summary>
/// <param name="grid">The grid instance which trying to learn</param>
/// <param name="A">The list of valid actions</param>
/// <param name="gamma">The discount factor</param>
/// <param name="alpha">The learning rate</param>
public SarsaLearning(Grid grid, List<Action> A, float gamma, float alpha) : base(grid, A, gamma, alpha) { }
/// <summary>
/// Construct a Lambda-learner instance
/// </summary>
/// <param name="grid">The grid instance which trying to learn</param>
/// <param name="A">The list of valid actions</param>
/// <param name="gamma">The discount factor</param>
/// <param name="alpha">The learning rate</param>
/// <param name="lambda">The lambda rate</param>
public RLambdaLearning(Grid grid, List<Action> A, float gamma, float alpha, float lambda) : base(grid, A, gamma, alpha) { this.Lambda = lambda; this.EligTable = new Hashtable(); }
/// <summary>
/// Construct a Q(lambda)-learner instance
/// </summary>
/// <param name="grid">The grid instance which trying to learn</param>
/// <param name="A">The list of valid actions</param>
/// <param name="gamma">The discount factor</param>
/// <param name="alpha">The learning rate</param>
/// <param name="lambda">The lambda factor</param>
public QLambdaLearning(Grid grid, List<Action> A, float gamma, float alpha, float lambda) : base(grid, A, gamma, lambda, alpha) { }
/// <summary>
/// Construct a TD(lambda)-learner instance
/// </summary>
/// <param name="grid">The grid instance which trying to learn</param>
/// <param name="A">The list of valid actions</param>
/// <param name="gamma">The discount factor</param>
/// <param name="alpha">The learning rate</param>
/// <param name="lambda">The lambda factor</param>
public TDLambda(Grid grid, List<Action> A, float gamma, float alpha, float lambda) : base(grid, A, gamma, lambda, alpha) { this.UTable = new Hashtable(); this.InitialState = grid.AgentPoint; }
private void loadToolStripMenuItem_Click(object sender, EventArgs e)
{
__enable_all_menus(false);
using (OpenFileDialog sfd = new OpenFileDialog())
{
sfd.DefaultExt = "dat";
sfd.AddExtension = true;
sfd.Filter = "Data files (*.dat)|*.dat";
#if !__DEBUG_PLOT__
var res = sfd.ShowDialog(this);
if (res == System.Windows.Forms.DialogResult.OK)
{
#else
sfd.FileName = "sarsa.dat";
#endif
BinaryFormatter bf = new BinaryFormatter();
{
using (var fs = sfd.OpenFile())
{
g = new Grid((Grid.BlockStatus[,])bf.Deserialize(fs), this.grid);
if (ql == null)
ql = new ReinforcementLearning.QLearning(
this.g,
new List<GridHelper.Directions>(Enum.GetValues(typeof(GridHelper.Directions)).Cast<GridHelper.Directions>()),
Properties.Settings.Default.Gamma,
Properties.Settings.Default.Alpha);
if (tdl == null)
tdl = new ReinforcementLearning.TDLambda(
ql,
Properties.Settings.Default.Lambda);
if (adp == null)
adp = new ReinforcementLearning.ADP(
ql);
ql.QTable = (Hashtable)bf.Deserialize(fs);
ql.VisitedStateActions = (Hashtable)bf.Deserialize(fs);
ql.StepCounter = (long)bf.Deserialize(fs);
// support for non-UTable contain files
if (fs.Position < fs.Length)
tdl.UTable = (Hashtable)bf.Deserialize(fs);
// support for non-UTable contain files
if (fs.Position < fs.Length)
this.TDLambdaUtilityProgress = (Hashtable)bf.Deserialize(fs);
// support for non-UTable contain files
if (fs.Position < fs.Length)
adp.UTable = (Hashtable)bf.Deserialize(fs);
// support for non-UTable contain files
if (fs.Position < fs.Length)
this.ADPUtilityProgress = (Hashtable)bf.Deserialize(fs);
}
}
__reload_grid();
__plot_policy(ql);
__plot_utility(tdl, adp);
this.toolStripStatus.Text = "The QTable saved successfully....";
#if !__DEBUG_PLOT__
}
#endif
}
__enable_all_menus(true);
}
private void newConfigurationToolStripMenuItem_Click(object sender, EventArgs e)
{
this.grid.CreateGraphics().Clear(Color.FromKnownColor(KnownColor.Control));
var s = Grid.GetSizeOfGrid(Properties.Settings.Default.GridSize);
g = new Grid(Properties.Settings.Default.GridSize);
this.Size = new Size(s.Width + 21, s.Height + 90);
Timer t = new Timer();
t.Interval = 100;
t.Tick += new EventHandler((_sender, _e) =>
{
t.Stop();
g.BlockBorders(this.grid);
g.Draw(this.grid.CreateGraphics());
__last_valid_grid_block = g.abs2grid(g.AgentPoint);
MarkStartPointGrid_Click(new object(), new EventArgs());
__last_valid_grid_block = g.abs2grid(g.GoalPoint);
MarkGoalPointGrid_Click(new object(), new EventArgs());
});
t.Start();
}
public GridForm()
{
InitializeComponent();
/**
* Draw the grid
*/
Size gridSize = Properties.Settings.Default.GridSize;
Size s = Grid.GetSizeOfGrid(gridSize);
this.grid.Image = new Bitmap(s.Width + 1, s.Height + 1, this.grid.CreateGraphics());
using (Graphics gfx = Graphics.FromImage(this.grid.Image))
{
g = new Grid(gridSize);
g.Draw(gfx);
this.Size = new Size(s.Width + 21, s.Height + 90);
}
/**
*
*/
this.Load += new EventHandler((sender, e) =>
{
Timer t = new Timer();
t.Interval = 100;
t.Tick += new EventHandler((_sender, _e) =>
{
t.Stop();
g.BlockBorders(this.grid);
});
t.Start();
});
this.KeyPreview = true;
this.KeyDown += new KeyEventHandler((sender, e) =>
{
switch (e.KeyCode)
{
case Keys.Escape: System.Diagnostics.Process.GetCurrentProcess().Kill(); break;
}
});
this.grid.MouseMove += new MouseEventHandler((sender, e) =>
{
if (this.__NearBySearchTimer != null) this.__NearBySearchTimer.Stop();
else
{
this.__NearBySearchTimer = new Timer();
this.__NearBySearchTimer.Interval = 15;
this.__NearBySearchTimer.Tick += new EventHandler((__sender, __e) =>
{
var p = (Point)this.__NearBySearchTimer.Tag;
if (g.GetNearByLineInfo(this.grid, p).Value.A != 0)
Cursor.Current = Cursors.Hand;
else Cursor.Current = Cursors.Default;
});
}
if (g.GetNearByLineInfo(this.grid, e.Location).Value.A != 0)
Cursor.Current = Cursors.Hand;
this.__NearBySearchTimer.Tag = e.Location;
this.__NearBySearchTimer.Start();
});
this.grid.MouseDown += new MouseEventHandler(grid_MouseDown);
this.toolStripStatus.TextChanged += new EventHandler((sende, e) =>
{
bool internal_change = false;
if (internal_change)
{
internal_change = false;
return;
}
if (this.toolStripStatus.Tag != null)
(this.toolStripStatus.Tag as Timer).Stop();
Timer t = new Timer();
t.Interval = 4000;
t.Tick += new EventHandler((_s, _e) =>
{
t.Stop();
internal_change = true;
this.toolStripStatus.Tag = null;
this.toolStripStatus.Text = "";
});
t.Start();
this.toolStripStatus.Tag = t;
});
if (System.IO.File.Exists("config.dat"))
loadConfigurationToolStripMenuItem_Click(new object(), new EventArgs());
else
newConfigurationToolStripMenuItem_Click(new object(), new EventArgs());
#if __DEBUG_PLOT__
loadToolStripMenuItem_Click(new object(), new EventArgs());
tDLambdaProgressShowToolStripMenuItem_Click(new object(), new EventArgs());
#endif
}
private void loadConfigurationToolStripMenuItem_Click(object sender, EventArgs e)
{
if (!System.IO.File.Exists("config.dat"))
{
MessageBox.Show("No configuration file found!");
this.toolStripStatus.Text = "Operation Abort...";
return;
}
BinaryFormatter bf = new BinaryFormatter();
{
using (System.IO.FileStream fs = new System.IO.FileStream("config.dat", System.IO.FileMode.Open, System.IO.FileAccess.Read, System.IO.FileShare.None))
{
g = new Grid((Grid.BlockStatus[,])bf.Deserialize(fs), this.grid);
g.AgentPoint = (Point)bf.Deserialize(fs);
g.GoalPoint = (Point)bf.Deserialize(fs);
}
__reload_grid();
}
this.toolStripStatus.Text = "Configuration Loaded Sucessfully...";
}