public QSearchResult Breadth_First(QState startState, bool output = false)
{
HashSet <QState> explored = new HashSet <QState>();
Dictionary <QState, decimal> bestSoFar = new Dictionary <QState, decimal>()
{
{ startState, 0 }
};
Queue <QState> toDo = new Queue <QState>();
toDo.Enqueue(startState);
Dictionary <QState, QSearchResult> pathTo = new Dictionary <QState, QSearchResult>()
{
{ startState, new QSearchResult() }
};
if (output)
{
WriteOutput("Searching for shortest path via Breadth-First Search...");
}
int steps = 0;
while (toDo.Any() && isRunning)
{
steps++;
QState current = toDo.Dequeue();
if (current.IsEnd())
{
if (output)
{
WriteOutput("Shortest path of " + pathTo[current].Count + " step(s) found after " + steps + " iteration(s).");
}
return(pathTo[current]);
}
else
{
explored.Add(current);
foreach (QAction action in current.GetActions())
{
QState newState = current.GetNewState(action);
if (!explored.Contains(newState))
{
decimal actualCost = bestSoFar[current] - current.GetValue();
if (!bestSoFar.ContainsKey(newState) || actualCost < bestSoFar[newState])
{
pathTo[newState] = new QSearchResult(pathTo[current]);
pathTo[newState].actionsList.Add(action);
pathTo[newState].QStatesList.Add(newState);
bestSoFar[newState] = actualCost;
toDo.Enqueue(newState);
}
}
}
}
}
if (output)
{
WriteOutput("No path found after " + steps + " iteration(s).");
}
return(null);
}
public QSearchResult AStar(QState startState, bool output = false, int maxQueue = 1000)
{
HashSet <QState> explored = new HashSet <QState>();
Dictionary <QState, decimal> bestSoFar = new Dictionary <QState, decimal>()
{
{ startState, 0 }
};
HeapPriorityQueue <QStateContainer> toDo = new HeapPriorityQueue <QStateContainer>(maxQueue);
toDo.Enqueue(new QStateContainer(startState), 0);
Dictionary <QState, QSearchResult> pathTo = new Dictionary <QState, QSearchResult>()
{
{ startState, new QSearchResult() }
};
if (output)
{
WriteOutput("Searching for shortest path via A-Star Search...");
}
int steps = 0;
while (toDo.Count > 0 && isRunning)
{
steps++;
QState current = toDo.Dequeue().qstate;
if (current.IsEnd())
{
if (output)
{
WriteOutput("Shortest path of " + pathTo[current].Count + " step(s) found after " + steps + " iteration(s).");
}
return(pathTo[current]);
}
else
{
explored.Add(current);
foreach (QAction action in current.GetActions())
{
QState newState = current.GetNewState(action);
if (!explored.Contains(newState))
{
decimal actualCost = bestSoFar[current] - current.GetValue();
if (!bestSoFar.ContainsKey(newState) || actualCost < bestSoFar[newState])
{
pathTo[newState] = new QSearchResult(pathTo[current]);
pathTo[newState].actionsList.Add(action);
pathTo[newState].QStatesList.Add(newState);
bestSoFar[newState] = actualCost;
toDo.Enqueue(new QStateContainer(newState), bestSoFar[newState] - 1 * newState.GetValueHeuristic());
}
}
}
}
}
if (output)
{
WriteOutput("No path found after " + steps + " iteration(s).");
}
return(null);
}
// Constructors ------------------------------------------------------------
public TestProbeProcess(HSTWorkcell workcell, string processID, string processName)
: base(workcell, processID, processName)
{
this._workcell = workcell;
_qTimer = new QTimer(this);
//RunInit mode
stateMoveToHomePosition = new QState(this.StateMoveToHomePosition);
statePublishSigTestProbeHomed = new QState(this.StatePublishSigTestProbeHomed);
stateWaitForSigStartBoundaryCheckTestProbeAxis = new QState(this.StateWaitForSigStartBoundaryCheckTestProbeAxis);
stateBoundaryCheck = new QState(this.StateBoundaryCheck);
stateRunInitCompleteMoveToParkPosition = new QState(this.StateRunInitCompleteMoveToParkPosition);
statePublishSigTestProbeBoundaryCheckComplete = new QState(this.StatePublishSigTestProbeBoundaryCheckComplete);
//Run mode
stateCheckProbeFunctionalTest = new QState(this.StateCheckProbeFunctionTest);
stateWaitForSigReadyForProbe = new QState(this.StateWaitForSigReadyForProbe);
stateMoveTestProbeToProbePosition = new QState(this.StateMoveTestProbeToProbePosition);
statePublishSigRequestForHGATesting = new QState(this.StatePublishSigRequestForHGATesting);
stateWaitForSigHGATestingDone = new QState(this.StateWaitForSigHGATestingDone);
stateMoveTestProbeToParkPosition = new QState(this.StateMoveTestProbeToParkPosition);
stateCheckDoubleTestPeriod = new QState(this.StateCheckDoubleTestPeriod);
stateStateWaitForSigGetMeasurementDone = new QState(this.StateWaitForSigGetMeasurementDone);
statePublishSigProbeDone = new QState(this.StatePublishSigProbeDone);
}
//Obtain the current state's super state
protected QState superstate(QState s)
{
QEvent e = new QEvent();
e.sig = (int)HSMSignals.SUPER_SIG;
return(s(e));
}
public QState GetState()
{
var p = PositionToState(transform.position);
var g = PositionToState(Goal.Position);
_grid.Populate(bounds => {
var ray = new Ray(new Vector3(bounds.center.x, 2, bounds.center.z), Vector3.down);
RaycastHit hit;
if (Physics.Raycast(ray, out hit, 3.0f))
{
return(hit.collider.gameObject == Goal.Instance.gameObject ? 1f : 0.1f);
}
return(0f);
});
var dead = !IsAboveGround();
var goal = p.SequenceEqual(g);
var line = VectorToGoal(transform.position, Goal.Position);
var v = line.normalized;
_linearState.At(0, v.x);
_linearState.At(1, v.z);
var terminal = dead || goal || DetectCycle();
var spatial = Matrix <float> .Build.Dense(1, 1, 1f);
var state = new QState(
new [] { spatial },
_linearState.Clone(),
goal ? 1 : 0,
terminal
);
return(state);
}
protected void UpdateLearningTable(int n, QState s, QAction a, decimal qv)
{
if (main != null)
{
main.UpdateLearningTable(n, s, a, qv);
}
}
// If deciding to toggle, call magnet scripts toggle functionality.
void FixedUpdate()
{
// Get current state
currentState = QState.getQState(magnet, opponents);
// Don't continue after magnet is done.
if (magnetScript.getCharge() != 0)
{
// Update Q value
if (lastState != null)
{
updateQValue();
}
// Choose an action
bool action = choseAction();
// Toggle if needed
if (action)
{
magnetScript.makeMove();
}
// Update memory
lastState = currentState;
lastAction = action;
lastPosition = magnet.GetComponent <Magnet>().getPosition();
}
}
public override QState Run(QState currentState, int trialNum, decimal learn, decimal discount, decimal explore)
{
QSearch qsearch = new QSearch(this);
QSearchResult actions = qsearch.Breadth_First(currentState, true);
if (actions != null)
{
foreach (QAction action in actions.actionsList)
{
if (!currentState.IsEnd() && isRunning && currentState.GetActions().Contains(action))
{
WriteOutput(currentState + ": " + action);
QState newState = currentState.GetNewState(action);
newState.Inherit(currentState);
newState.Step();
currentState = newState;
}
}
if (currentState.IsEnd()) WriteOutput(currentState + ": End");
else
{
WriteOutput("Existing solution no longer applicable. Re-solving...");
return Run(currentState, trialNum, learn, discount, explore);
}
}
else WriteOutput("No solution found.", true);
return currentState;
}
private QState AO_StateSearch_InState_IfBlock_Else(QEvent e)
{
switch (e.sig)
{
case (int)HSMSignals.ENTRY_SIG:
case (int)HSMSignals.EXIT_SIG:
{
return(handled);
}
case (int)Signals.NEW_WORD_SIG:
{
string tempWord = ((WordFeeder)e).Word;
lastState = AO_StateSearch_InState;
if (tempWord.Equals("if"))
{
transition(AO_StateSearch_InState_IfBlock);
}
else if (tempWord.Contains("return") && ifBlockScopeLevel == scopeLevel)
{
ifTransition = new Transition(ifBlockCondition, null);
transition(AO_StateSearchs_InState_GuardedTransitionReturnFound);
}
else
{
transition(AO_StateSearch_InState_IfBlock_WaitReturn);
}
return(handled);
}
}
return(AO_StateSearch_InState_IfBlock);
}
//the key learning algorithem, current is TD(0)
public void UpdateKnowledge(QState CurQState)
{
State CurState = CurQState.StartState;
Action CurAction = CurQState.CurrentAction;
State NextState = CurQState.EndState;
float Reward = CurQState.Reward;
if (Experience.ContainsKey(CurState) == false)
{
Experience[CurState] = new StateActionValue();
}
Experience[CurState].TotalActionCount += 1;
if (Experience.ContainsKey(NextState))
{
Experience[CurState].ActionValue[CurAction] += LeaningRate * (Reward + DiscountRate * Experience[NextState].MaxValue - Experience[CurState].ActionValue[CurAction]);
}
else
{
Experience[CurState].ActionValue[CurAction] += LeaningRate * (Reward - Experience[CurState].ActionValue[CurAction]);
}
Experience[CurState].ActionCount[CurAction] += 1;
Debug.Log("Action: " + CurQState.CurrentAction.ToString() + "State: (" + CurState.PlayerNpcDistance.ToString() + "," + CurState.PlayerNpcEulerAngle.ToString() + ")"
+ " Reward: " + Reward.ToString() + " Utility: " + Experience[CurState].ActionValue[CurAction].ToString() + " ActionCount: " + Experience[CurState].ActionCount[CurAction].ToString());
if (Experience[CurState].ActionValue[CurAction] > Experience[CurState].MaxValue || Experience[CurState].MaxAction == Action.None)
{
Experience[CurState].MaxAction = CurAction;
Experience[CurState].MaxValue = Experience[CurState].ActionValue[CurAction];
}
}
public override QState Run(QState currentState, int trialNum, decimal learn, decimal discount, decimal explore)
{
QSearch qsearch = new QSearch(this);
QSearchResult actions = qsearch.Depth_First(currentState, true);
if (actions != null)
{
foreach (QAction action in actions.actionsList)
{
if (!currentState.IsEnd() && isRunning && currentState.GetActions().Contains(action))
{
WriteOutput(currentState + ": " + action);
QState newState = currentState.GetNewState(action);
newState.Inherit(currentState);
newState.Step();
currentState = newState;
}
}
if (currentState.IsEnd())
{
WriteOutput(currentState + ": End");
}
else
{
WriteOutput("Existing solution no longer applicable. Re-solving...");
return(Run(currentState, trialNum, learn, discount, explore));
}
}
else
{
WriteOutput("No solution found.", true);
}
return(currentState);
}
// Constructors ------------------------------------------------------------
public InputEEProcess(HSTWorkcell workcell, string processID, string processName)
: base(workcell, processID, processName)
{
this._workcell = workcell;
_qTimer = new QTimer(this);
//RunInit mode
stateRunInitMoveToHome = new QState(this.StateRunInitMoveToHome);
statePublishInputEEHomed = new QState(this.StatePublishInputEEHomed);
stateWaitForSigStartBoundaryCheckInputEEAxis = new QState(this.StateWaitForSigStartBoundaryCheckInputEEAxis);
stateBoundaryCheck = new QState(this.StateBoundaryCheck);
statePublishInputEEBoundaryCheckComplete = new QState(this.StatePublishInputEEBoundaryCheckComplete);
stateCheckVaccuum = new QState(this.StateCheckVaccuum);
stateRunInitCompleteMoveToParkPosition = new QState(this.StateRunInitCompleteMoveToParkPosition);
//Run mode
stateRunStart = new QState(this.StateRunStart);
stateRunCheckHGAReadyForPick = new QState(this.StateRunCheckHGAReadyForPick);
statePickHgaVacuumOn = new QState(this.StatePickHgaVacuumOn);
statePickHgaZDown = new QState(this.StatePickHgaZDown);
statePickHgaZUpToParkPosition = new QState(this.StatePickHgaZUpToParkPosition);
statePickHgaCheckVacuum = new QState(this.StatePickHgaCheckVacuum);
statePickHgaPublishPickDone = new QState(this.StatePickHgaPublishPickDone);
stateRunCheckHGAReadyForPlace = new QState(this.StateRunCheckHGAReadyForPlace);
statePlaceHgaZDown = new QState(this.StatePlaceHgaZDown);
statePlaceHgaVacuumOff = new QState(this.StatePlaceHgaVacuumOff);
stateWaitPrecisorTurnOnAllVacuums = new QState(this.StateWaitPrecisorTurnOnAllVacuums);
statePlaceHgaZUpToParkPosition = new QState(this.StatePlaceHgaZUpToParkPosition);
statePlaceHgaPublishPlaceDone = new QState(this.StatePlaceHgaPublishPlaceDone);
statePerformCleaningWithDycem = new QState(this.StatePerformCleaningWithDycem);
}
protected virtual QState S_TimeKeeping(IQEvent ev)
{
switch (ev.QSignal)
{
case QSignals.Init: {
LogStateEvent(StateLogType.Init, s_TimeKeeping, s_TimeKeeping_Time);
InitializeState(s_TimeKeeping_Time);
} return(null);
case QSignals.Entry: {
LogStateEvent(StateLogType.Entry, s_TimeKeeping);
_TimeKeeping_DeepHistory = s_TimeKeeping;
} return(null);
case QSignals.Exit: {
LogStateEvent(StateLogType.Exit, s_TimeKeeping);
} return(null);
case QualifiedSampleWatchSignals.SetEvt: {
LogStateEvent(StateLogType.EventTransition, s_TimeKeeping, s_Setting, "SetEvt", "SetEvt");
TransitionTo(s_Setting, s_trans_SetEvt_TimeKeeping_2_Setting);
return(null);
} // SetEvt
} // switch
return(TopState);
} // S_TimeKeeping
// Constructors ------------------------------------------------------------
public InputTurnStationProcess(HSTWorkcell workcell, string processID, string processName)
: base(workcell, processID, processName)
{
this._workcell = workcell;
// HSM delegates
_qTimer = new QTimer(this);
//RunInit mode
stateStandbyInputTurnSection = new QState(this.StateStandbyInputTurnSection);
//Run mode
stateRunStart = new QState(this.StateRunStart);
stateDoAutoClearCarrier = new QState(this.StateDoAutoClearCarrier);
stateCheckForNextCarrier = new QState(this.StateCheckForNextCarrier);
stateReleaseCarrierToInputTurnStation = new QState(this.StateReleaseCarrierToInputTurnStation);
stateInputTurnSectionTurnForward = new QState(this.StateInputTurnSectionTurnForward);
stateVisionInspectCarrierAfterLoad = new QState(this.StateVisionInspectCarrierAfterLoad);
stateWaitForSigInputStationReady = new QState(this.StateWaitForSigInputStationReady);
statePublishSigCarrierInInputStation = new QState(this.StatePublishSigCarrierInInputStation);
stateReleaseCarrier = new QState(this.StateReleaseCarrier);
stateInputTurnSectionTurnBackward = new QState(this.StateInputTurnSectionTurnBackward);
stateReportTICError = new QState(this.StateReportTICError);
stateReportCRDLError = new QState(this.StateReportCRDLError);
stateReportGetputError = new QState(this.StateReportGetputError);
}
// Constructors ------------------------------------------------------------
public OutputEEProcess(HSTWorkcell workcell, string processID, string processName)
: base(workcell, processID, processName)
{
this._workcell = workcell;
_qTimer = new QTimer(this);
//RunInit mode
stateMoveToHomePosition = new QState(this.StateMoveToHomePosition);
statePublishOutputEEHomed = new QState(this.StatePublishOutputEEHomed);
stateWaitForSigStartBoundaryCheckOutputEEAxis = new QState(this.StateWaitForSigStartBoundaryCheckOutputEEAxis);
stateBoundaryCheck = new QState(this.StateBoundaryCheck);
statePublishOutputEEBoundaryCheckComplete = new QState(this.StatePublishOutputEEBoundaryCheckComplete);
stateRunInitCompleteMoveToParkPosition = new QState(this.StateRunInitCompleteMoveToParkPosition);
//Run mode
stateWaitForPrecisorReadyForPick = new QState(this.StateWaitForPrecisorReadyForPick);
statePickHGAVacuumOn = new QState(this.StatePickHGAVacuumOn);
statePickHGAZDown = new QState(this.StatePickHGAZDown);
statePickHGACheckVacuumAndMoveUp = new QState(this.StatePickHGACheckVacuumAndMoveUp);
stateAlertOperatorHGADrop = new QState(this.StateAlertOperatorHGADrop);
statePublishSigPrecisorPickDone = new QState(this.StatePublishSigPrecisorPickDone);
stateWaitForSigOutputCarrierReadyForPlace = new QState(this.StateWaitForSigOutputCarrierReadyForPlace);
statePlaceHGAZDown = new QState(this.StatePlaceHGAZDown);
statePlaceHGAVacuumOff = new QState(this.StatePlaceHGAVacuumOff);
statePlaceHGAZUp = new QState(this.StatePlaceHGAZUp);
statePublishSigOutputCarrierPlaceDone = new QState(this.StatePublishSigOutputCarrierPlaceDone);
statePerformCleaningWithDycem = new QState(this.StatePerformCleaningWithDycem);
}
protected virtual QState S_Setting(IQEvent ev)
{
switch (ev.QSignal)
{
case QSignals.Init: {
LogStateEvent(StateLogType.Init, s_Setting, s_Setting_Hour);
InitializeState(s_Setting_Hour);
} return(null);
case QSignals.Entry: {
LogStateEvent(StateLogType.Entry, s_Setting, "Blink()");
Blink();
} return(null);
case QSignals.Exit: {
NoBlink();
LogStateEvent(StateLogType.Exit, s_Setting, "NoBlink()");
} return(null);
case QualifiedSampleWatchSignals.SetEvt: {
QState toState_TimeKeeping = _TimeKeeping_DeepHistory == null ? s_TimeKeeping : _TimeKeeping_DeepHistory;
LogStateEvent(StateLogType.EventTransition, s_Setting, toState_TimeKeeping, "SetEvt", "SetEvt");
TransitionTo(toState_TimeKeeping);
return(null);
} // SetEvt
} // switch
return(TopState);
} // S_Setting
public static Action GetAction(QState state)
{
if (_instance == null)
{
throw new InvalidOperationException("Manager in invalid state, call InitAgent first");
}
switch (_instance.Mode)
{
case QAIMode.Learning:
if (_instance._sceneIsOver)
{
return () => {}
}
;
return(_instance._qlearning.GetLearningAction(state) ?? _instance.EndOfEpisode);
case QAIMode.Testing:
return(() => _instance.TesterAction(state));
case QAIMode.Runnning:
return(_instance._qlearning.GreedyPolicy(state).Action);
default:
return(() => {});
}
}
public static Dictionary <QAction, float> Query(QState state)
{
var q = _instance._qlearning.Q(state);
return(_instance._qlearning.Actions
.ToDictionary(a => a, a => q(a)));
}
protected virtual QState S_TimeKeeping_Time(IQEvent ev)
{
switch (ev.QSignal)
{
case QSignals.Entry: {
LogStateEvent(StateLogType.Entry, s_TimeKeeping_Time);
_TimeKeeping_DeepHistory = s_TimeKeeping_Time;
SetTimeOut("TimeKeeping_Time_t0_TickEvt", TimeSpan.FromSeconds(1), new QEvent("TickEvt"), TimeOutType.Repeat);
} return(null);
case QSignals.Exit: {
ClearTimeOut("TimeKeeping_Time_t0_TickEvt");
LogStateEvent(StateLogType.Exit, s_TimeKeeping_Time);
} return(null);
case QualifiedSampleWatchSignals.ModeEvt: {
LogStateEvent(StateLogType.EventTransition, s_TimeKeeping_Time, s_TimeKeeping_Date, "ModeEvt", "ModeEvt");
TransitionTo(s_TimeKeeping_Date, s_trans_ModeEvt_TimeKeeping_Time_2_TimeKeeping_Date);
return(null);
} // ModeEvt
case QualifiedSampleWatchSignals.TickEvt: {
TickTime();
LogStateEvent(StateLogType.EventTransition, s_TimeKeeping_Time, s_TimeKeeping_Time, "TickEvt", "t0-every 1 raise TickEvt/TickTime()");
TransitionTo(s_TimeKeeping_Time, s_trans_t0_TickEvt_TimeKeeping_Time_2_TimeKeeping_Time);
return(null);
} // TickEvt
} // switch
return(s_TimeKeeping);
} // S_TimeKeeping_Time
protected override QState ActiveTest_Running(IQEvent qEvent)
{
QState result = base.ActiveTest_Running(qEvent);
if (result == null)
{
return(null);
}
else
{
switch (qEvent.QSignal)
{
case (int)QFSignal.TestStep:
NextStep();
return(null);
// DESIGN NOTE: Wait for the associated ActiveTest to complete...
case (int)QFSignal.TestResult:
if (string.IsNullOrEmpty(((TestEvent)qEvent).Reason))
{
TransitionToPassed();
}
else
{
TransitionToFailed(((TestEvent)qEvent).Reason);
}
return(null);
}
return(result);
}
}
//Go to a state upon initial transition
protected void initial_transition(QState s)
{
if (state == (QState)superstate(s))
{
state = s;
}
}
public override QState GetRandNextState(QState prevState)
{
var rng = new Random();
var rand = rng.NextDouble();
Cell result = null;
var possibilites = this.GetAvailableStates();
while (result == null)
{
if (0 <= rand && rand < 0.25 && !NorthWall)
{
result = Cells[NeighborNorthID];
}
if (0.25 <= rand && rand < 0.5 && !SouthWall)
{
result = Cells[NeighborSouthID];
}
if (0.5 <= rand && rand < 0.75 && !WestWall)
{
result = Cells[NeighborWestID];
}
if (0.75 <= rand && rand <= 1 && !EastWall)
{
result = Cells[NeighborEastID];
}
if (prevState != null && result != null && result.Equals((Cell)prevState) && rng.NextDouble() >= 0.6)
{
result = null;
}
rand = rng.NextDouble();
}
return(result);
}
private void ArchiveState(QState state)
{
if (_history.Count >= MaxHistorySize)
{
_history.RemoveLast();
}
_history.AddFirst(state);
}
private string StateNameFrom(QState state)
{
if (state == null)
{
return("NULLSTATE");
}
return(state.Method.Name);
}
} //init
private Reminder()
{
Polling = new QState(this.DoPolling);
Processing = new QState(this.DoProcessing);
Idle = new QState(this.DoIdle);
Busy = new QState(this.DoBusy);
Final = new QState(this.DoFinal);
} //ctor
private float CalcQValue(QState qState)
{
float QValue = _wDistToGhost1 * DistanceToGhost(qState.PacmanCoordinates, qState.GhostCoordinates[0]) +
_wDistToGhost2 * DistanceToGhost(qState.PacmanCoordinates, qState.GhostCoordinates[1]) +
_wDistToFood * DistanceToNearestFood(qState.PacmanCoordinates, qState.FoodCoordinates);
return(QValue);
}
public BattleBeatenState(GameObject player, GameObject npc)
{
stateID = StateID.BattleBeaten;
CurPlayer = player;
CurNpc = npc;
anim = npc.GetComponent <Animator>();
MDPQstate = new QState(Action.Beaten);
}
/// <summary>
/// Создание экземпляра класса <see cref="QChar"/>.
/// </summary>
public QChar(char chr, QState state, int eulerEquals, int eulerRoundings, int popularity)
{
State = state;
EulerEquals = eulerEquals;
EulerRoundings = eulerRoundings;
Popularity = popularity;
Char = chr;
}
// Constructors & Finalizers -------------------------------------------
public ImageFileManageProcess(HSTWorkcell workcell, string processID, string processName)
: base(workcell, processID, processName)
{
// initialize HSM delegates
_isNonIdleProcess = true;
stateStartRunning = new QState(this.StartRunning);
_stateTimer = new QTimer(this);
}
protected virtual void QUpdate(int n, QState currentState, QAction action, QState newState, decimal reward)
{
QStateActionPair p = new QStateActionPair(currentState, action);
decimal maxQ = GetMaxValue(newState);
QValues[p] = (1 - learn) * GetQValue(p) + learn * (reward + discount * maxQ);
UpdateLearningTable(n, currentState, action, QValues[p]);
}
protected virtual void CreateStateFields()
{
if (null == _MyStaticInstance)
{
s_NoSparks = new QState(S_NoSparks);
s_Sparking = new QState(S_Sparking);
}
}
/// <summary>
/// Default constructor - initializes all fields to default values
/// </summary>
public QHsmTest()
{
m_s0 = new QState(this.s0);
m_s1 = new QState(this.s1);
m_s11 = new QState(this.s11);
m_s2 = new QState(this.s2);
m_s21 = new QState(this.s21);
m_s211 = new QState(this.s211);
}
public QAgent(QAlgo algo, QState initialState)
{
trialNum = 0;
this.initialState = initialState;
initialState.setQAgent(this);
currentAlgo = algo;
currentAlgo.setQAgent(this);
currentAlgo.Initialize();
}
public Philosopher(int philosopherId)
{
m_PhilosopherId= philosopherId;
m_StateThinking = new QState(this.Thinking);
m_StateHungry = new QState(this.Hungry);
m_StateEating = new QState(this.Eating);
m_Timer = new QTimer(this);
}
// This runs a single trial/instance of the QState problem. QLearner will automatically run many times for learning or once to apply what has been learned.
// Must return the final state
public override QState Run(QState currentState, int trialNum, decimal learn, decimal discount, decimal explore)
{
while (!currentState.IsEnd() && currentState.GetActions().Length > 0 && isRunning)
{
QAction action = currentState.GetActions().ElementAt(r.Next(currentState.GetActions().Length));
QState newState = currentState.GetNewState(action);
newState.Inherit(currentState);
newState.Step();
WriteOutput((CurrentMode == LEARN ? "Trial " + trialNum + ", " : "") + ": '" + action + "' @ " + currentState.ToString());
currentState = newState;
}
return currentState;
}
public Table(int numberOfPhilosophers)
{
m_NumberOfPhilosophers = numberOfPhilosophers;
m_ForkIsUsed = new bool[m_NumberOfPhilosophers];
m_PhilosopherIsHungry = new bool[m_NumberOfPhilosophers];
for(int i = 0; i < m_NumberOfPhilosophers; i++)
{
m_ForkIsUsed[i] = false;
m_PhilosopherIsHungry[i] = false;
}
m_StateServing = new QState(this.Serving);
}
private AlarmClock()
{
TimeKeeping = new QState(this.DoTimeKeeping);
Mode12Hour = new QState(this.DoMode12Hour);
Mode24Hour = new QState(this.DoMode24Hour);
Final = new QState(this.DoFinal);
// Create the delegate that invokes methods for the timer.
timerDelegate = new TimerCallback(GenerateTimerTickEvent);
myAlarm = Alarm.Instance;//we could instead just use "Alarm.Instance" and eliminate "myAlarm"
#if USE_DOTNET_EVENTS
myAlarm.AlarmActivated += new AlarmClock.AlarmDisplayHandler(AlarmActivated);
#endif
}
public override QState Run(QState currentState, int trialNum, decimal learn, decimal discount, decimal explore)
{
this.learn = learn; this.discount = discount; this.explore = explore;
decimal score = 0;
int actionsTaken = 0;
while (!currentState.IsEnd() && GetOutcomes(currentState).Count > 0 && isRunning)
{
actionsTaken++;
QAction a;
bool exp;
if (explore > 0 && (decimal)random.NextDouble() <= explore)
{
a = GetRandomAction(currentState);
exp = true;
}
else
{
a = GetBestAction(currentState);
exp = false;
}
QState newState = currentState.GetNewState(a);
WriteOutput((CurrentMode == LEARN ? "Trial " + trialNum + ", " : "") + "#" + actionsTaken + " " + (exp ? "Explore" : "Action") + ": '" + a + "' @ " + currentState.ToString());
newState.Inherit(currentState);
newState.Step();
decimal r = GetReward(currentState, newState);
score += r;
QUpdate(actionsTaken, currentState, a, newState, r);
WriteOutput((CurrentMode == LEARN ? "Trial " + trialNum + ", " : "") + "#" + actionsTaken + " Gain " + Math.Round(r, 4) + ", Total " + Math.Round(score, 4));
foreach (KeyValuePair<QStateActionPair, QState> kv in newState.GetObservedStates(currentState, a))
{
QState observedPriorState = kv.Key.state;
QAction observedAction = kv.Key.action;
QState observedState = kv.Value;
decimal observedR = GetReward(observedPriorState, observedState);
QUpdate(actionsTaken, observedPriorState, observedAction, observedState, observedR);
WriteOutput((CurrentMode == LEARN ? "Trial " + trialNum + ", " : "") + "#" + actionsTaken + " Observed: '" + observedAction + "' @ " + observedPriorState.ToString() + " | Gain " + Math.Round(observedR, 4));
}
currentState = newState;
}
if (isRunning)
{
WriteOutput("Trial " + trialNum + ": " + Math.Round(score, 4) + " in " + actionsTaken + " step" + (actionsTaken == 1 ? "" : "s") + ".");
}
return currentState;
}
// This runs a single trial/instance of the QState problem. QLearner will automatically run many times for learning or once to apply what has been learned.
// Must return the final state
public virtual QState Run(QState currentState, int trialNum, decimal learn, decimal discount, decimal explore)
{
/* Should follow the following format:
while (!currentState.IsEnd() && currentState.GetActions().Length > 0 && isRunning)
{
*Pick an action for the current state*
...
QState newState = currentState.GetNewState(a);
newState.Inherit(currentState);
newState.Step();
...
*Learn from the reward/cost of the decision*
...
currentState = newState;
}
* */
return currentState;
}
public QSearchResult AStar(QState startState, bool output = false, int maxQueue = 1000)
{
HashSet<QState> explored = new HashSet<QState>();
Dictionary<QState, decimal> bestSoFar = new Dictionary<QState, decimal>() { { startState, 0 } };
HeapPriorityQueue<QStateContainer> toDo = new HeapPriorityQueue<QStateContainer>(maxQueue);
toDo.Enqueue(new QStateContainer(startState), 0);
Dictionary<QState, QSearchResult> pathTo = new Dictionary<QState, QSearchResult>() { { startState, new QSearchResult() } };
if (output) WriteOutput("Searching for shortest path via A-Star Search...");
int steps = 0;
while (toDo.Count > 0 && isRunning)
{
steps++;
QState current = toDo.Dequeue().qstate;
if (current.IsEnd())
{
if (output) WriteOutput("Shortest path of " + pathTo[current].Count + " step(s) found after " + steps + " iteration(s).");
return pathTo[current];
}
else
{
explored.Add(current);
foreach (QAction action in current.GetActions())
{
QState newState = current.GetNewState(action);
if (!explored.Contains(newState))
{
decimal actualCost = bestSoFar[current] - current.GetValue();
if (!bestSoFar.ContainsKey(newState) || actualCost < bestSoFar[newState])
{
pathTo[newState] = new QSearchResult(pathTo[current]);
pathTo[newState].actionsList.Add(action);
pathTo[newState].QStatesList.Add(newState);
bestSoFar[newState] = actualCost;
toDo.Enqueue(new QStateContainer(newState), bestSoFar[newState] - 1 * newState.GetValueHeuristic());
}
}
}
}
}
if (output) WriteOutput("No path found after " + steps + " iteration(s).");
return null;
}
public QSearchResult Breadth_First(QState startState, bool output = false)
{
HashSet<QState> explored = new HashSet<QState>();
Dictionary<QState, decimal> bestSoFar = new Dictionary<QState,decimal>(){{startState, 0}};
Queue<QState> toDo = new Queue<QState>();
toDo.Enqueue(startState);
Dictionary<QState, QSearchResult> pathTo = new Dictionary<QState, QSearchResult>() { { startState, new QSearchResult() } };
if(output) WriteOutput("Searching for shortest path via Breadth-First Search...");
int steps = 0;
while (toDo.Any() && isRunning)
{
steps++;
QState current = toDo.Dequeue();
if (current.IsEnd())
{
if(output) WriteOutput("Shortest path of " + pathTo[current].Count + " step(s) found after " + steps + " iteration(s).");
return pathTo[current];
}
else
{
explored.Add(current);
foreach (QAction action in current.GetActions())
{
QState newState = current.GetNewState(action);
if (!explored.Contains(newState))
{
decimal actualCost = bestSoFar[current] - current.GetValue();
if (!bestSoFar.ContainsKey(newState) || actualCost < bestSoFar[newState])
{
pathTo[newState] = new QSearchResult(pathTo[current]);
pathTo[newState].actionsList.Add(action);
pathTo[newState].QStatesList.Add(newState);
bestSoFar[newState] = actualCost;
toDo.Enqueue(newState);
}
}
}
}
}
if(output) WriteOutput("No path found after " + steps + " iteration(s).");
return null;
}
// Update the weights of each feature based on their contribution to the reward
protected override void QUpdate(int n, QState currentState, QAction action, QState newState, decimal reward)
{
decimal maxQ = GetMaxValue(newState);
QStateActionPair p = new QStateActionPair(currentState, action);
Dictionary<QFeature, decimal> features;
if (featureCache.ContainsKey(p)) features = featureCache[p];
else
{
features = currentState.GetFeatures(action);
featureCache[p] = features;
}
decimal currentQ = GetQValue(p);
decimal difference = reward + discount * maxQ - currentQ;
foreach (KeyValuePair<QFeature, decimal> feature in features)
{
try
{
if (!QWeights.ContainsKey(feature.Key)) QWeights[feature.Key] = 0;
decimal oldWeight = QWeights[feature.Key];
decimal newWeight = oldWeight + learn * difference * feature.Value;
if (Math.Abs(newWeight) <= 1000000)
{
QWeights[feature.Key] = newWeight;
}
else WriteOutput("Warning: Weights diverging. Check that your features are valid and measured consistently with everything else.", true);
}
catch (Exception e)
{
WriteOutput("Exception: " + e + "\n*Check that your features are valid and measured consistently with everything else.*", true);
Abort();
break;
}
}
// Output
foreach (QFeature f in features.Keys)
{
UpdateLearningTable(-1, f.ToString(), QWeights[f].ToString(), features[f]);
}
}
protected virtual void RaiseFinalStateReached(EventHandler handler, ILQHsm hsm, QState state)
{
if (handler != null)
{
LogStateEventArgs logEvent = new LogStateEventArgs ();
logEvent.LogType = StateLogType.Final;
logEvent.State = state;
handler (hsm, logEvent);
}
}
public override void Open(object o, QState initialState)
{
QWeights = (Dictionary<QFeature, decimal>)o;
ClearLearningTable();
int i = 1;
foreach (KeyValuePair<QFeature, decimal> kv in QWeights)
UpdateLearningTable(i++, kv.Key.ToString(), kv.Value.ToString(), 0);
}
/// <summary>
/// 添加状态
/// </summary>
/// <param name="state">State.</param>
public void AddState(string name)
{
StateDict [name] = new QState(name);
}
public override void Open(object o, QState initialState)
{
QValues.Clear();
ClearLearningTable();
foreach (KeyValuePair<object[], decimal> kv in ((Dictionary<object[], decimal>)o))
{
QValues.Add(new QStateActionPair(initialState.Open(kv.Key[0]), (QAction)kv.Key[1]), kv.Value);
}
int i=1;
foreach (KeyValuePair<QStateActionPair, decimal> kv in QValues)
{
UpdateLearningTable(i++, kv.Key.state, kv.Key.action, kv.Value);
}
}
public QAgent(QLearner master, QAlgo algo, QState initialState)
: this(algo, initialState)
{
this.master = master;
}
// Return a random action for exploration
protected virtual QAction GetRandomAction(QState state)
{
IEnumerable<QAction> s = GetOutcomes(state).Select(x => x.action);
return s.ElementAt(random.Next(s.Count()));
}
/// <summary>
/// Default constructor - initializes all fields to default values
/// </summary>
public QHsmBase1()
{
m_s0 = new QState(this.s0);
m_s01 = new QState(this.s01);
m_s02 = new QState(this.s02);
}
// Return best action to take from current state (best QValue or random if tied)
protected virtual QAction GetBestAction(QState state)
{
decimal maxVal = GetMaxValue(state);
IEnumerable<QAction> s = GetOutcomes(state).Where(x => GetQValue(x) == maxVal).Select(x => x.action);
int n = s.Count();
if (n == 1) return s.First();
else
{
return s.ElementAt(random.Next(n));
}
}
// Return the value of the best action taken at given state, else 0 if not known
protected virtual decimal GetMaxValue(QState state)
{
return GetOutcomes(state).Select(x => GetQValue(x)).Max();
}
// Return list of all possible states that can result from an action taken from the current state
protected virtual List<QStateActionPair> GetOutcomes(QState state)
{
return state.GetActions().Select(x => new QStateActionPair(state, x)).ToList() ;
}
public virtual decimal GetReward(QState currentState, QState newState)
{
return newState.GetValue() - currentState.GetValue();
}
private void LoadQStatePlugin(string f)
{
try
{
bool loaded = false;
foreach(Assembly a in AppDomain.CurrentDomain.GetAssemblies())
foreach (Type qstate in a.GetTypes().
Where(t => String.Equals(t.Namespace, "QLearner.QStates", StringComparison.Ordinal)).
Where(t => t.IsSubclassOf(typeof(QState))).
Where(t => t.IsVisible).
Where(t => !t.IsAbstract).
Where(t => t.Name==f))
{
state = (QState)Activator.CreateInstance(qstate);
WriteOutput("Loaded QState: " + qstate.Name, true);
state.setQLearner(this);
Settings.Enabled = state.HasSettings;
if (state != null && algo != null)
{
agent = new QAgent(this, algo, state);
Reset();
Awaken.Enabled = Learn.Enabled = true;
}
Properties.Settings.Default.QStatePlugin = qstate.Name;
Properties.Settings.Default.Save();
loaded = true;
break;
}
if (!loaded)
{
WriteOutput("QState Not Found: " + f, true);
ResetPlugins(false);
}
}
catch (Exception e)
{
WriteOutput("Unable to load QState: " + f +"\n"+e, true);
ResetPlugins(false);
}
}
public void UpdateLearningTable(int n, QState s, QAction a, decimal qv)
{
if (InvokeRequired)
{
Invoke(new UpdateLearningTableD(UpdateLearningTable), n, s, a, qv);
}
else
{
QStateActionPair p = new QStateActionPair(s, a);
if (LearningTableQStateKeys.ContainsKey(p))
{
LearningTableQStateKeys[p].Cells["Num"].Value = n;
LearningTableQStateKeys[p].Cells["QValue"].Value = qv;
}
else
{
LearningTable.Rows.Add(n, s.ToString(), a.ToString(), qv);
LearningTableQStateKeys[p] = LearningTable.Rows[LearningTable.Rows.Count - 1]; ;
}
}
}
public QStateActionPair(QState s, QAction a)
{
state = s;
action = a;
}
private QState SlowTwo; //step 2 -- part 2 of long op that user may want to cancel
#endregion Fields
#region Constructors
private RunToCompletion()
{
Dispatching = new QState(this.DoDispatching);
CantInterrupt = new QState(this.DoCantInterrupt);
Interruptible = new QState(this.DoInterruptible);
SlowOne = new QState(this.DoSlowOne);
SlowTwo = new QState(this.DoSlowTwo);
Completed = new QState(this.DoCompleted);
Final = new QState(this.DoFinal);
}
/// <summary>
/// Default constructor - initializes all fields to default values
/// </summary>
public QHsmDerived3()
{
m_s021 = new QState(this.s021);
}
// The below functions are called to open and save files holding what has been learned. QLearner will handle the file write/read operations and receive or pass and general object to the QAlgo. Make sure what you pass in Save is a serializable object. If what you save includes QState, use the Open from the initialState to rebuild the state.
public virtual void Open(object o, QState initialState)
{
}
protected virtual void QUpdate(int n, QState currentState, QAction action, QState newState, decimal reward)
{
QStateActionPair p = new QStateActionPair(currentState, action);
decimal maxQ = GetMaxValue(newState);
QValues[p] = (1 - learn) * GetQValue(p) + learn * (reward + discount * maxQ);
UpdateLearningTable(n, currentState, action, QValues[p]);
}
