public void Test_QLearning_Path_Finder() { // start var master = new MDPState(2); var kitchen = new MDPState(3); master.Successors.Add(new MDPSuccessorState(new AI.Action(1, "Goto Kitchen"), 0.1, kitchen, 0)); var entrance = new MDPState(1); var lounge = new MDPState(4); kitchen.Successors.Add(new MDPSuccessorState(new AI.Action(2, "Goto Lounge"), 0.1, lounge, -15)); kitchen.Successors.Add(new MDPSuccessorState(new AI.Action(3, "Goto Entrance Hall"), 0, entrance, -30)); var spare = new MDPState(0); lounge.Successors.Add(new MDPSuccessorState(new AI.Action(4, "Goto Spare Room"), 0.1, spare, -10)); var outside = new MDPState(5); lounge.Successors.Add(new MDPSuccessorState(new AI.Action(5, "Go Outside"), 0.1, outside, 30)); entrance.Successors.Add(new MDPSuccessorState(new AI.Action(6, "Go Outside"), 0.1, outside, 50)); outside.Successors.Add(new MDPSuccessorState(new AI.Action(7, "Stay Outside"), 0.2, outside, 50)); var examples = MDPConverter.ToExamples(master); Assert.Equal(7, examples.Item1.Rows); Assert.Equal(7, examples.Item2.Length); Assert.Equal(7, examples.Item3.Rows); Assert.Equal(7, examples.Item4.Length); var generator = new Reinforcement.QLearning.QLearnerGenerator() { Lambda = 0.9 }; Reinforcement.QLearning.QLearnerModel model = (Reinforcement.QLearning.QLearnerModel) generator.Generate(examples.Item1, examples.Item2, examples.Item3, examples.Item4); Assert.Equal(3, (int) model.Predict(kitchen.ToVector())/*, "Expected to move from kitchen to entrance hall"*/); Assert.Equal(5, (int) model.Predict(lounge.ToVector())/*, "Expected to move from lounge to outside"*/); Assert.Equal(7, (int) model.Predict(outside.ToVector())/*, "Expected to stay outside"*/); string path = "Start: " + master.Id; IMDPState current = master; int counter = 0; while (current.Id != outside.Id) { if (counter > 20) break; double v = model.Predict(current.ToVector()); var next = current.GetSuccessors().Where(w => w.Action.Id == (int) v).FirstOrDefault() as IMDPSuccessor; if (next == null) break; current = next.State as IMDPState; counter++; path += $"\n next: { current.Id } ({ next.Reward.ToString("N2") })"; } Console.Write(path); }
public void Test_QLearning_Path_Finder() { // start var master = new MDPState(2); var kitchen = new MDPState(3); master.Successors.Add(new MDPSuccessorState(new AI.Action(1, "Goto Kitchen"), 0.1, kitchen, 0)); var entrance = new MDPState(1); var lounge = new MDPState(4); kitchen.Successors.Add(new MDPSuccessorState(new AI.Action(2, "Goto Lounge"), 0.1, lounge, -15)); kitchen.Successors.Add(new MDPSuccessorState(new AI.Action(3, "Goto Entrance Hall"), 0, entrance, -30)); var spare = new MDPState(0); lounge.Successors.Add(new MDPSuccessorState(new AI.Action(4, "Goto Spare Room"), 0.1, spare, -10)); var outside = new MDPState(5); lounge.Successors.Add(new MDPSuccessorState(new AI.Action(5, "Go Outside"), 0.1, outside, 30)); entrance.Successors.Add(new MDPSuccessorState(new AI.Action(6, "Go Outside"), 0.1, outside, 50)); outside.Successors.Add(new MDPSuccessorState(new AI.Action(7, "Stay Outside"), 0.2, outside, 50)); var examples = MDPConverter.ToExamples(master); Assert.Equal(7, examples.Item1.Rows); Assert.Equal(7, examples.Item2.Length); Assert.Equal(7, examples.Item3.Rows); Assert.Equal(7, examples.Item4.Length); var generator = new Reinforcement.QLearning.QLearnerGenerator() { Lambda = 0.9 }; Reinforcement.QLearning.QLearnerModel model = (Reinforcement.QLearning.QLearnerModel)generator.Generate(examples.Item1, examples.Item2, examples.Item3, examples.Item4); Assert.Equal(3, (int)model.Predict(kitchen.ToVector()) /*, "Expected to move from kitchen to entrance hall"*/); Assert.Equal(5, (int)model.Predict(lounge.ToVector()) /*, "Expected to move from lounge to outside"*/); Assert.Equal(7, (int)model.Predict(outside.ToVector()) /*, "Expected to stay outside"*/); string path = "Start: " + master.Id; IMDPState current = master; int counter = 0; while (current.Id != outside.Id) { if (counter > 20) { break; } double v = model.Predict(current.ToVector()); var next = current.GetSuccessors().Where(w => w.Action.Id == (int)v).FirstOrDefault() as IMDPSuccessor; if (next == null) { break; } current = next.State as IMDPState; counter++; path += $"\n next: { current.Id } ({ next.Reward.ToString("N2") })"; } Console.Write(path); }