static void passiveTDAgentDemo()
{
CellWorld <double> cw = CellWorldFactory.CreateCellWorldForFig17_1();
CellWorldEnvironment cwe = new CellWorldEnvironment(
cw.GetCellAt(1, 1),
cw.GetCells(),
MDPFactory.createTransitionProbabilityFunctionForFigure17_1(cw),
CommonFactory.CreateRandom());
IMap <Cell <double>, CellWorldAction> fixedPolicy = CollectionFactory.CreateMap <Cell <double>, CellWorldAction>();
fixedPolicy.Put(cw.GetCellAt(1, 1), CellWorldAction.Up);
fixedPolicy.Put(cw.GetCellAt(1, 2), CellWorldAction.Up);
fixedPolicy.Put(cw.GetCellAt(1, 3), CellWorldAction.Right);
fixedPolicy.Put(cw.GetCellAt(2, 1), CellWorldAction.Left);
fixedPolicy.Put(cw.GetCellAt(2, 3), CellWorldAction.Right);
fixedPolicy.Put(cw.GetCellAt(3, 1), CellWorldAction.Left);
fixedPolicy.Put(cw.GetCellAt(3, 2), CellWorldAction.Up);
fixedPolicy.Put(cw.GetCellAt(3, 3), CellWorldAction.Right);
fixedPolicy.Put(cw.GetCellAt(4, 1), CellWorldAction.Left);
PassiveTDAgent <Cell <double>, CellWorldAction> ptda
= new PassiveTDAgent <Cell <double>, CellWorldAction>(fixedPolicy, 0.2, 1.0);
cwe.AddAgent(ptda);
output_utility_learning_rates(ptda, 20, 500, 100, 1);
}
public static void Main(params string[] args)
{
IRandom r = CommonFactory.CreateRandom();
EightPuzzleBoard board = new EightPuzzleBoard(new int[] { 0, 1, 2, 3,
4, 5, 6, 7, 8 });
for (int i = 0; i < 50; ++i)
{
int th = r.Next(4);
if (th == 0)
{
board.moveGapUp();
}
if (th == 1)
{
board.moveGapDown();
}
if (th == 2)
{
board.moveGapLeft();
}
if (th == 3)
{
board.moveGapRight();
}
}
System.Console.WriteLine(board);
}
public override void AddAgent(IAgent a)
{
int idx = CommonFactory.CreateRandom().Next(locations.Size());
envState.setAgentLocation(a, locations.Get(idx));
base.AddAgent(a);
}
public static Individual <int> generateRandomIndividual(int boardSize)
{
ICollection <int> individualRepresentation = CollectionFactory.CreateQueue <int>();
for (int i = 0; i < boardSize; ++i)
{
individualRepresentation.Add(CommonFactory.CreateRandom().Next(boardSize));
}
return(new Individual <int>(individualRepresentation));
}
/// <summary>
/// Generate matrix with random elements
/// </summary>
/// <param name="m">Number of rows.</param>
/// <param name="n">Number of colums.</param>
/// <returns>An m-by-n matrix with uniformly distributed random elements.</returns>
public static Matrix Random(int m, int n)
{
IRandom _random = CommonFactory.CreateRandom();
Matrix A = new Matrix(m, n);
double[,] X = A.GetArray();
for (int i = 0; i < m; ++i)
{
for (int j = 0; j < n; j++)
{
X[i, j] = _random.NextDouble();
}
}
return(A);
}
static void qLearningAgentDemo()
{
CellWorld <double> cw = CellWorldFactory.CreateCellWorldForFig17_1();
CellWorldEnvironment cwe = new CellWorldEnvironment(
cw.GetCellAt(1, 1),
cw.GetCells(),
MDPFactory.createTransitionProbabilityFunctionForFigure17_1(cw),
CommonFactory.CreateRandom());
QLearningAgent <Cell <double>, CellWorldAction> qla = new QLearningAgent <Cell <double>, CellWorldAction>(
MDPFactory.createActionsFunctionForFigure17_1(cw),
CellWorldAction.None, 0.2, 1.0, 5,
2.0);
cwe.AddAgent(qla);
output_utility_learning_rates(qla, 20, 10000, 500, 20);
}
/**
* Creates a board with <code>size</code> rows and size columns. Column and
* row indices start with 0.
*
* @param config
* Controls whether the board is initially empty or contains some
* queens.
*/
public NQueensBoard(int size, Config config)
: this(size)
{
if (config == Config.QUEENS_IN_FIRST_ROW)
{
for (int i = 0; i < size; ++i)
{
addQueenAt(new XYLocation(i, 0));
}
}
else if (config == Config.QUEEN_IN_EVERY_COL)
{
IRandom r = CommonFactory.CreateRandom();
for (int i = 0; i < size; ++i)
{
addQueenAt(new XYLocation(i, r.Next(size)));
}
}
}
/**
* Execute the agent action
*/
public override void executeAction(IAgent a, IAction action)
{
if (ACTION_MOVE_RIGHT == action)
{
envState.setAgentLocation(a, LOCATION_B);
updatePerformanceMeasure(a, -1);
}
else if (ACTION_MOVE_LEFT == action)
{
envState.setAgentLocation(a, LOCATION_A);
updatePerformanceMeasure(a, -1);
}
else if (ACTION_SUCK == action)
{
// case: square is dirty
if (VacuumEnvironment.LocationState.Dirty == envState.getLocationState(envState.getAgentLocation(a)))
{
string currentLocation = envState.getAgentLocation(a);
string adjacentLocation = (currentLocation.Equals("A")) ? "B" : "A";
// always clean current square
envState.setLocationState(currentLocation, VacuumEnvironment.LocationState.Clean);
// possibly clean adjacent square
if (CommonFactory.CreateRandom().NextDouble() > 0.5)
{
envState.setLocationState(adjacentLocation, VacuumEnvironment.LocationState.Clean);
}
} // case: square is clean
else if (VacuumEnvironment.LocationState.Clean == envState.getLocationState(envState.getAgentLocation(a)))
{
// possibly dirty current square
if (random.NextBoolean())
{
envState.setLocationState(envState.getAgentLocation(a), VacuumEnvironment.LocationState.Dirty);
}
}
}
else if (action.IsNoOp())
{
_isDone = true;
}
}
static void passiveADPAgentDemo()
{
System.Console.WriteLine("=======================");
System.Console.WriteLine("DEMO: Passive-ADP-Agent");
System.Console.WriteLine("=======================");
System.Console.WriteLine("Figure 21.3");
System.Console.WriteLine("-----------");
CellWorld <double> cw = CellWorldFactory.CreateCellWorldForFig17_1();
CellWorldEnvironment cwe = new CellWorldEnvironment(
cw.GetCellAt(1, 1),
cw.GetCells(),
MDPFactory.createTransitionProbabilityFunctionForFigure17_1(cw),
CommonFactory.CreateRandom());
IMap <Cell <double>, CellWorldAction> fixedPolicy = CollectionFactory.CreateInsertionOrderedMap <Cell <double>, CellWorldAction>();
fixedPolicy.Put(cw.GetCellAt(1, 1), CellWorldAction.Up);
fixedPolicy.Put(cw.GetCellAt(1, 2), CellWorldAction.Up);
fixedPolicy.Put(cw.GetCellAt(1, 3), CellWorldAction.Right);
fixedPolicy.Put(cw.GetCellAt(2, 1), CellWorldAction.Left);
fixedPolicy.Put(cw.GetCellAt(2, 3), CellWorldAction.Right);
fixedPolicy.Put(cw.GetCellAt(3, 1), CellWorldAction.Left);
fixedPolicy.Put(cw.GetCellAt(3, 2), CellWorldAction.Up);
fixedPolicy.Put(cw.GetCellAt(3, 3), CellWorldAction.Right);
fixedPolicy.Put(cw.GetCellAt(4, 1), CellWorldAction.Left);
PassiveADPAgent <Cell <double>, CellWorldAction> padpa = new PassiveADPAgent <Cell <double>, CellWorldAction>(
fixedPolicy, cw.GetCells(), cw.GetCellAt(1, 1),
MDPFactory.createActionsFunctionForFigure17_1(cw),
new ModifiedPolicyEvaluation <Cell <double>, CellWorldAction>(10, 1.0));
cwe.AddAgent(padpa);
output_utility_learning_rates(padpa, 20, 100, 100, 1);
System.Console.WriteLine("=========================");
}
public PriorSample()
: this(CommonFactory.CreateRandom())
{
}
public GibbsAsk()
: this(CommonFactory.CreateRandom())
{
}
//
// PRIVATE METHODS
//
// if /\E > 0 then current <- next
// else current <- next only with probability e^(/\E/T)
private bool shouldAccept(double temperature, double deltaE)
{
return((deltaE > 0.0) ||
(CommonFactory.CreateRandom().NextDouble() <= probabilityOfAcceptance(temperature, deltaE)));
}
public LikelihoodWeighting()
: this(CommonFactory.CreateRandom())
{
}
protected static void output_utility_learning_rates(
ReinforcementAgent <Cell <double>, CellWorldAction> reinforcementAgent,
int numRuns, int numTrialsPerRun, int rmseTrialsToReport,
int reportEveryN)
{
if (rmseTrialsToReport > (numTrialsPerRun / reportEveryN))
{
throw new IllegalArgumentException("Requesting to report too many RMSE trials, max allowed for args is "
+ (numTrialsPerRun / reportEveryN));
}
CellWorld <double> cw = CellWorldFactory.CreateCellWorldForFig17_1();
CellWorldEnvironment cwe = new CellWorldEnvironment(
cw.GetCellAt(1, 1),
cw.GetCells(),
MDPFactory.createTransitionProbabilityFunctionForFigure17_1(cw),
CommonFactory.CreateRandom());
cwe.AddAgent(reinforcementAgent);
IMap <int, ICollection <IMap <Cell <double>, double> > > runs = CollectionFactory.CreateInsertionOrderedMap <int, ICollection <IMap <Cell <double>, double> > >();
for (int r = 0; r < numRuns; r++)
{
reinforcementAgent.reset();
ICollection <IMap <Cell <double>, double> > trials = CollectionFactory.CreateQueue <IMap <Cell <double>, double> >();
for (int t = 0; t < numTrialsPerRun; t++)
{
cwe.executeTrial();
if (0 == t % reportEveryN)
{
IMap <Cell <double>, double> u = reinforcementAgent
.getUtility();
//if (null == u.Get(cw.getCellAt(1, 1)))
//{
// throw new IllegalStateException(
// "Bad Utility State Encountered: r=" + r
// + ", t=" + t + ", u=" + u);
//}
trials.Add(u);
}
}
runs.Put(r, trials);
}
IStringBuilder v4_3 = TextFactory.CreateStringBuilder();
IStringBuilder v3_3 = TextFactory.CreateStringBuilder();
IStringBuilder v1_3 = TextFactory.CreateStringBuilder();
IStringBuilder v1_1 = TextFactory.CreateStringBuilder();
IStringBuilder v3_2 = TextFactory.CreateStringBuilder();
IStringBuilder v2_1 = TextFactory.CreateStringBuilder();
for (int t = 0; t < (numTrialsPerRun / reportEveryN); t++)
{
// Use the last run
IMap <Cell <double>, double> u = runs.Get(numRuns - 1).Get(t);
v4_3.Append((u.ContainsKey(cw.GetCellAt(4, 3)) ? u.Get(cw
.GetCellAt(4, 3)) : 0.0) + "\t");
v3_3.Append((u.ContainsKey(cw.GetCellAt(3, 3)) ? u.Get(cw
.GetCellAt(3, 3)) : 0.0) + "\t");
v1_3.Append((u.ContainsKey(cw.GetCellAt(1, 3)) ? u.Get(cw
.GetCellAt(1, 3)) : 0.0) + "\t");
v1_1.Append((u.ContainsKey(cw.GetCellAt(1, 1)) ? u.Get(cw
.GetCellAt(1, 1)) : 0.0) + "\t");
v3_2.Append((u.ContainsKey(cw.GetCellAt(3, 2)) ? u.Get(cw
.GetCellAt(3, 2)) : 0.0) + "\t");
v2_1.Append((u.ContainsKey(cw.GetCellAt(2, 1)) ? u.Get(cw
.GetCellAt(2, 1)) : 0.0) + "\t");
}
IStringBuilder rmseValues = TextFactory.CreateStringBuilder();
for (int t = 0; t < rmseTrialsToReport; t++)
{
// Calculate the Root Mean Square Error for utility of 1,1
// for this trial# across all runs
double xSsquared = 0;
for (int r = 0; r < numRuns; r++)
{
IMap <Cell <double>, double> u = runs.Get(r).Get(t);
double val1_1 = u.Get(cw.GetCellAt(1, 1));
//if (null == val1_1)
//{
// throw new IllegalStateException(
// "U(1,1,) is not present: r=" + r + ", t=" + t
// + ", runs.size=" + runs.Size()
// + ", runs(r).Size()=" + runs.Get(r).Size()
// + ", u=" + u);
//}
xSsquared += System.Math.Pow(0.705 - val1_1, 2);
}
double rmse = System.Math.Sqrt(xSsquared / runs.Size());
rmseValues.Append(rmse);
rmseValues.Append("\t");
}
System.Console
.WriteLine("Note: You may copy and paste the following lines into a spreadsheet to generate graphs of learning rate and RMS error in utility:");
System.Console.WriteLine("(4,3)" + "\t" + v4_3);
System.Console.WriteLine("(3,3)" + "\t" + v3_3);
System.Console.WriteLine("(1,3)" + "\t" + v1_3);
System.Console.WriteLine("(1,1)" + "\t" + v1_1);
System.Console.WriteLine("(3,2)" + "\t" + v3_2);
System.Console.WriteLine("(2,1)" + "\t" + v2_1);
System.Console.WriteLine("RMSeiu" + "\t" + rmseValues);
}
/**
* Construct a Particle Filtering instance.
*
* @param N
* the number of samples to be maintained
* @param dbn
* a DBN with prior <b>P</b>(<b>X</b><sub>0</sub>), transition
* model <b>P</b>(<b>X</b><sub>1</sub> | <b>X</b><sub>0</sub>),
* sensor model <b>P</b>(<b>E</b><sub>1</sub> |
* <b>X</b><sub>1</sub>)
*/
public ParticleFiltering(int N, IDynamicBayesianNetwork dbn)
: this(N, dbn, CommonFactory.CreateRandom())
{
}
public GeneticAlgorithm(int individualLength, ICollection <A> finiteAlphabet, double mutationProbability)
: this(individualLength, finiteAlphabet, mutationProbability, CommonFactory.CreateRandom())
{
}
