public QSearchResult Breadth_First(QState startState, bool output = false) { HashSet <QState> explored = new HashSet <QState>(); Dictionary <QState, decimal> bestSoFar = new Dictionary <QState, decimal>() { { startState, 0 } }; Queue <QState> toDo = new Queue <QState>(); toDo.Enqueue(startState); Dictionary <QState, QSearchResult> pathTo = new Dictionary <QState, QSearchResult>() { { startState, new QSearchResult() } }; if (output) { WriteOutput("Searching for shortest path via Breadth-First Search..."); } int steps = 0; while (toDo.Any() && isRunning) { steps++; QState current = toDo.Dequeue(); if (current.IsEnd()) { if (output) { WriteOutput("Shortest path of " + pathTo[current].Count + " step(s) found after " + steps + " iteration(s)."); } return(pathTo[current]); } else { explored.Add(current); foreach (QAction action in current.GetActions()) { QState newState = current.GetNewState(action); if (!explored.Contains(newState)) { decimal actualCost = bestSoFar[current] - current.GetValue(); if (!bestSoFar.ContainsKey(newState) || actualCost < bestSoFar[newState]) { pathTo[newState] = new QSearchResult(pathTo[current]); pathTo[newState].actionsList.Add(action); pathTo[newState].QStatesList.Add(newState); bestSoFar[newState] = actualCost; toDo.Enqueue(newState); } } } } } if (output) { WriteOutput("No path found after " + steps + " iteration(s)."); } return(null); }
public QSearchResult AStar(QState startState, bool output = false, int maxQueue = 1000) { HashSet <QState> explored = new HashSet <QState>(); Dictionary <QState, decimal> bestSoFar = new Dictionary <QState, decimal>() { { startState, 0 } }; HeapPriorityQueue <QStateContainer> toDo = new HeapPriorityQueue <QStateContainer>(maxQueue); toDo.Enqueue(new QStateContainer(startState), 0); Dictionary <QState, QSearchResult> pathTo = new Dictionary <QState, QSearchResult>() { { startState, new QSearchResult() } }; if (output) { WriteOutput("Searching for shortest path via A-Star Search..."); } int steps = 0; while (toDo.Count > 0 && isRunning) { steps++; QState current = toDo.Dequeue().qstate; if (current.IsEnd()) { if (output) { WriteOutput("Shortest path of " + pathTo[current].Count + " step(s) found after " + steps + " iteration(s)."); } return(pathTo[current]); } else { explored.Add(current); foreach (QAction action in current.GetActions()) { QState newState = current.GetNewState(action); if (!explored.Contains(newState)) { decimal actualCost = bestSoFar[current] - current.GetValue(); if (!bestSoFar.ContainsKey(newState) || actualCost < bestSoFar[newState]) { pathTo[newState] = new QSearchResult(pathTo[current]); pathTo[newState].actionsList.Add(action); pathTo[newState].QStatesList.Add(newState); bestSoFar[newState] = actualCost; toDo.Enqueue(new QStateContainer(newState), bestSoFar[newState] - 1 * newState.GetValueHeuristic()); } } } } } if (output) { WriteOutput("No path found after " + steps + " iteration(s)."); } return(null); }
// Constructors ------------------------------------------------------------ public TestProbeProcess(HSTWorkcell workcell, string processID, string processName) : base(workcell, processID, processName) { this._workcell = workcell; _qTimer = new QTimer(this); //RunInit mode stateMoveToHomePosition = new QState(this.StateMoveToHomePosition); statePublishSigTestProbeHomed = new QState(this.StatePublishSigTestProbeHomed); stateWaitForSigStartBoundaryCheckTestProbeAxis = new QState(this.StateWaitForSigStartBoundaryCheckTestProbeAxis); stateBoundaryCheck = new QState(this.StateBoundaryCheck); stateRunInitCompleteMoveToParkPosition = new QState(this.StateRunInitCompleteMoveToParkPosition); statePublishSigTestProbeBoundaryCheckComplete = new QState(this.StatePublishSigTestProbeBoundaryCheckComplete); //Run mode stateCheckProbeFunctionalTest = new QState(this.StateCheckProbeFunctionTest); stateWaitForSigReadyForProbe = new QState(this.StateWaitForSigReadyForProbe); stateMoveTestProbeToProbePosition = new QState(this.StateMoveTestProbeToProbePosition); statePublishSigRequestForHGATesting = new QState(this.StatePublishSigRequestForHGATesting); stateWaitForSigHGATestingDone = new QState(this.StateWaitForSigHGATestingDone); stateMoveTestProbeToParkPosition = new QState(this.StateMoveTestProbeToParkPosition); stateCheckDoubleTestPeriod = new QState(this.StateCheckDoubleTestPeriod); stateStateWaitForSigGetMeasurementDone = new QState(this.StateWaitForSigGetMeasurementDone); statePublishSigProbeDone = new QState(this.StatePublishSigProbeDone); }
//Obtain the current state's super state protected QState superstate(QState s) { QEvent e = new QEvent(); e.sig = (int)HSMSignals.SUPER_SIG; return(s(e)); }
public QState GetState() { var p = PositionToState(transform.position); var g = PositionToState(Goal.Position); _grid.Populate(bounds => { var ray = new Ray(new Vector3(bounds.center.x, 2, bounds.center.z), Vector3.down); RaycastHit hit; if (Physics.Raycast(ray, out hit, 3.0f)) { return(hit.collider.gameObject == Goal.Instance.gameObject ? 1f : 0.1f); } return(0f); }); var dead = !IsAboveGround(); var goal = p.SequenceEqual(g); var line = VectorToGoal(transform.position, Goal.Position); var v = line.normalized; _linearState.At(0, v.x); _linearState.At(1, v.z); var terminal = dead || goal || DetectCycle(); var spatial = Matrix <float> .Build.Dense(1, 1, 1f); var state = new QState( new [] { spatial }, _linearState.Clone(), goal ? 1 : 0, terminal ); return(state); }
protected void UpdateLearningTable(int n, QState s, QAction a, decimal qv) { if (main != null) { main.UpdateLearningTable(n, s, a, qv); } }
// If deciding to toggle, call magnet scripts toggle functionality. void FixedUpdate() { // Get current state currentState = QState.getQState(magnet, opponents); // Don't continue after magnet is done. if (magnetScript.getCharge() != 0) { // Update Q value if (lastState != null) { updateQValue(); } // Choose an action bool action = choseAction(); // Toggle if needed if (action) { magnetScript.makeMove(); } // Update memory lastState = currentState; lastAction = action; lastPosition = magnet.GetComponent <Magnet>().getPosition(); } }
public override QState Run(QState currentState, int trialNum, decimal learn, decimal discount, decimal explore) { QSearch qsearch = new QSearch(this); QSearchResult actions = qsearch.Breadth_First(currentState, true); if (actions != null) { foreach (QAction action in actions.actionsList) { if (!currentState.IsEnd() && isRunning && currentState.GetActions().Contains(action)) { WriteOutput(currentState + ": " + action); QState newState = currentState.GetNewState(action); newState.Inherit(currentState); newState.Step(); currentState = newState; } } if (currentState.IsEnd()) WriteOutput(currentState + ": End"); else { WriteOutput("Existing solution no longer applicable. Re-solving..."); return Run(currentState, trialNum, learn, discount, explore); } } else WriteOutput("No solution found.", true); return currentState; }
private QState AO_StateSearch_InState_IfBlock_Else(QEvent e) { switch (e.sig) { case (int)HSMSignals.ENTRY_SIG: case (int)HSMSignals.EXIT_SIG: { return(handled); } case (int)Signals.NEW_WORD_SIG: { string tempWord = ((WordFeeder)e).Word; lastState = AO_StateSearch_InState; if (tempWord.Equals("if")) { transition(AO_StateSearch_InState_IfBlock); } else if (tempWord.Contains("return") && ifBlockScopeLevel == scopeLevel) { ifTransition = new Transition(ifBlockCondition, null); transition(AO_StateSearchs_InState_GuardedTransitionReturnFound); } else { transition(AO_StateSearch_InState_IfBlock_WaitReturn); } return(handled); } } return(AO_StateSearch_InState_IfBlock); }
//the key learning algorithem, current is TD(0) public void UpdateKnowledge(QState CurQState) { State CurState = CurQState.StartState; Action CurAction = CurQState.CurrentAction; State NextState = CurQState.EndState; float Reward = CurQState.Reward; if (Experience.ContainsKey(CurState) == false) { Experience[CurState] = new StateActionValue(); } Experience[CurState].TotalActionCount += 1; if (Experience.ContainsKey(NextState)) { Experience[CurState].ActionValue[CurAction] += LeaningRate * (Reward + DiscountRate * Experience[NextState].MaxValue - Experience[CurState].ActionValue[CurAction]); } else { Experience[CurState].ActionValue[CurAction] += LeaningRate * (Reward - Experience[CurState].ActionValue[CurAction]); } Experience[CurState].ActionCount[CurAction] += 1; Debug.Log("Action: " + CurQState.CurrentAction.ToString() + "State: (" + CurState.PlayerNpcDistance.ToString() + "," + CurState.PlayerNpcEulerAngle.ToString() + ")" + " Reward: " + Reward.ToString() + " Utility: " + Experience[CurState].ActionValue[CurAction].ToString() + " ActionCount: " + Experience[CurState].ActionCount[CurAction].ToString()); if (Experience[CurState].ActionValue[CurAction] > Experience[CurState].MaxValue || Experience[CurState].MaxAction == Action.None) { Experience[CurState].MaxAction = CurAction; Experience[CurState].MaxValue = Experience[CurState].ActionValue[CurAction]; } }
public override QState Run(QState currentState, int trialNum, decimal learn, decimal discount, decimal explore) { QSearch qsearch = new QSearch(this); QSearchResult actions = qsearch.Depth_First(currentState, true); if (actions != null) { foreach (QAction action in actions.actionsList) { if (!currentState.IsEnd() && isRunning && currentState.GetActions().Contains(action)) { WriteOutput(currentState + ": " + action); QState newState = currentState.GetNewState(action); newState.Inherit(currentState); newState.Step(); currentState = newState; } } if (currentState.IsEnd()) { WriteOutput(currentState + ": End"); } else { WriteOutput("Existing solution no longer applicable. Re-solving..."); return(Run(currentState, trialNum, learn, discount, explore)); } } else { WriteOutput("No solution found.", true); } return(currentState); }
// Constructors ------------------------------------------------------------ public InputEEProcess(HSTWorkcell workcell, string processID, string processName) : base(workcell, processID, processName) { this._workcell = workcell; _qTimer = new QTimer(this); //RunInit mode stateRunInitMoveToHome = new QState(this.StateRunInitMoveToHome); statePublishInputEEHomed = new QState(this.StatePublishInputEEHomed); stateWaitForSigStartBoundaryCheckInputEEAxis = new QState(this.StateWaitForSigStartBoundaryCheckInputEEAxis); stateBoundaryCheck = new QState(this.StateBoundaryCheck); statePublishInputEEBoundaryCheckComplete = new QState(this.StatePublishInputEEBoundaryCheckComplete); stateCheckVaccuum = new QState(this.StateCheckVaccuum); stateRunInitCompleteMoveToParkPosition = new QState(this.StateRunInitCompleteMoveToParkPosition); //Run mode stateRunStart = new QState(this.StateRunStart); stateRunCheckHGAReadyForPick = new QState(this.StateRunCheckHGAReadyForPick); statePickHgaVacuumOn = new QState(this.StatePickHgaVacuumOn); statePickHgaZDown = new QState(this.StatePickHgaZDown); statePickHgaZUpToParkPosition = new QState(this.StatePickHgaZUpToParkPosition); statePickHgaCheckVacuum = new QState(this.StatePickHgaCheckVacuum); statePickHgaPublishPickDone = new QState(this.StatePickHgaPublishPickDone); stateRunCheckHGAReadyForPlace = new QState(this.StateRunCheckHGAReadyForPlace); statePlaceHgaZDown = new QState(this.StatePlaceHgaZDown); statePlaceHgaVacuumOff = new QState(this.StatePlaceHgaVacuumOff); stateWaitPrecisorTurnOnAllVacuums = new QState(this.StateWaitPrecisorTurnOnAllVacuums); statePlaceHgaZUpToParkPosition = new QState(this.StatePlaceHgaZUpToParkPosition); statePlaceHgaPublishPlaceDone = new QState(this.StatePlaceHgaPublishPlaceDone); statePerformCleaningWithDycem = new QState(this.StatePerformCleaningWithDycem); }
protected virtual QState S_TimeKeeping(IQEvent ev) { switch (ev.QSignal) { case QSignals.Init: { LogStateEvent(StateLogType.Init, s_TimeKeeping, s_TimeKeeping_Time); InitializeState(s_TimeKeeping_Time); } return(null); case QSignals.Entry: { LogStateEvent(StateLogType.Entry, s_TimeKeeping); _TimeKeeping_DeepHistory = s_TimeKeeping; } return(null); case QSignals.Exit: { LogStateEvent(StateLogType.Exit, s_TimeKeeping); } return(null); case QualifiedSampleWatchSignals.SetEvt: { LogStateEvent(StateLogType.EventTransition, s_TimeKeeping, s_Setting, "SetEvt", "SetEvt"); TransitionTo(s_Setting, s_trans_SetEvt_TimeKeeping_2_Setting); return(null); } // SetEvt } // switch return(TopState); } // S_TimeKeeping
// Constructors ------------------------------------------------------------ public InputTurnStationProcess(HSTWorkcell workcell, string processID, string processName) : base(workcell, processID, processName) { this._workcell = workcell; // HSM delegates _qTimer = new QTimer(this); //RunInit mode stateStandbyInputTurnSection = new QState(this.StateStandbyInputTurnSection); //Run mode stateRunStart = new QState(this.StateRunStart); stateDoAutoClearCarrier = new QState(this.StateDoAutoClearCarrier); stateCheckForNextCarrier = new QState(this.StateCheckForNextCarrier); stateReleaseCarrierToInputTurnStation = new QState(this.StateReleaseCarrierToInputTurnStation); stateInputTurnSectionTurnForward = new QState(this.StateInputTurnSectionTurnForward); stateVisionInspectCarrierAfterLoad = new QState(this.StateVisionInspectCarrierAfterLoad); stateWaitForSigInputStationReady = new QState(this.StateWaitForSigInputStationReady); statePublishSigCarrierInInputStation = new QState(this.StatePublishSigCarrierInInputStation); stateReleaseCarrier = new QState(this.StateReleaseCarrier); stateInputTurnSectionTurnBackward = new QState(this.StateInputTurnSectionTurnBackward); stateReportTICError = new QState(this.StateReportTICError); stateReportCRDLError = new QState(this.StateReportCRDLError); stateReportGetputError = new QState(this.StateReportGetputError); }
// Constructors ------------------------------------------------------------ public OutputEEProcess(HSTWorkcell workcell, string processID, string processName) : base(workcell, processID, processName) { this._workcell = workcell; _qTimer = new QTimer(this); //RunInit mode stateMoveToHomePosition = new QState(this.StateMoveToHomePosition); statePublishOutputEEHomed = new QState(this.StatePublishOutputEEHomed); stateWaitForSigStartBoundaryCheckOutputEEAxis = new QState(this.StateWaitForSigStartBoundaryCheckOutputEEAxis); stateBoundaryCheck = new QState(this.StateBoundaryCheck); statePublishOutputEEBoundaryCheckComplete = new QState(this.StatePublishOutputEEBoundaryCheckComplete); stateRunInitCompleteMoveToParkPosition = new QState(this.StateRunInitCompleteMoveToParkPosition); //Run mode stateWaitForPrecisorReadyForPick = new QState(this.StateWaitForPrecisorReadyForPick); statePickHGAVacuumOn = new QState(this.StatePickHGAVacuumOn); statePickHGAZDown = new QState(this.StatePickHGAZDown); statePickHGACheckVacuumAndMoveUp = new QState(this.StatePickHGACheckVacuumAndMoveUp); stateAlertOperatorHGADrop = new QState(this.StateAlertOperatorHGADrop); statePublishSigPrecisorPickDone = new QState(this.StatePublishSigPrecisorPickDone); stateWaitForSigOutputCarrierReadyForPlace = new QState(this.StateWaitForSigOutputCarrierReadyForPlace); statePlaceHGAZDown = new QState(this.StatePlaceHGAZDown); statePlaceHGAVacuumOff = new QState(this.StatePlaceHGAVacuumOff); statePlaceHGAZUp = new QState(this.StatePlaceHGAZUp); statePublishSigOutputCarrierPlaceDone = new QState(this.StatePublishSigOutputCarrierPlaceDone); statePerformCleaningWithDycem = new QState(this.StatePerformCleaningWithDycem); }
protected virtual QState S_Setting(IQEvent ev) { switch (ev.QSignal) { case QSignals.Init: { LogStateEvent(StateLogType.Init, s_Setting, s_Setting_Hour); InitializeState(s_Setting_Hour); } return(null); case QSignals.Entry: { LogStateEvent(StateLogType.Entry, s_Setting, "Blink()"); Blink(); } return(null); case QSignals.Exit: { NoBlink(); LogStateEvent(StateLogType.Exit, s_Setting, "NoBlink()"); } return(null); case QualifiedSampleWatchSignals.SetEvt: { QState toState_TimeKeeping = _TimeKeeping_DeepHistory == null ? s_TimeKeeping : _TimeKeeping_DeepHistory; LogStateEvent(StateLogType.EventTransition, s_Setting, toState_TimeKeeping, "SetEvt", "SetEvt"); TransitionTo(toState_TimeKeeping); return(null); } // SetEvt } // switch return(TopState); } // S_Setting
public static Action GetAction(QState state) { if (_instance == null) { throw new InvalidOperationException("Manager in invalid state, call InitAgent first"); } switch (_instance.Mode) { case QAIMode.Learning: if (_instance._sceneIsOver) { return () => {} } ; return(_instance._qlearning.GetLearningAction(state) ?? _instance.EndOfEpisode); case QAIMode.Testing: return(() => _instance.TesterAction(state)); case QAIMode.Runnning: return(_instance._qlearning.GreedyPolicy(state).Action); default: return(() => {}); } }
public static Dictionary <QAction, float> Query(QState state) { var q = _instance._qlearning.Q(state); return(_instance._qlearning.Actions .ToDictionary(a => a, a => q(a))); }
protected virtual QState S_TimeKeeping_Time(IQEvent ev) { switch (ev.QSignal) { case QSignals.Entry: { LogStateEvent(StateLogType.Entry, s_TimeKeeping_Time); _TimeKeeping_DeepHistory = s_TimeKeeping_Time; SetTimeOut("TimeKeeping_Time_t0_TickEvt", TimeSpan.FromSeconds(1), new QEvent("TickEvt"), TimeOutType.Repeat); } return(null); case QSignals.Exit: { ClearTimeOut("TimeKeeping_Time_t0_TickEvt"); LogStateEvent(StateLogType.Exit, s_TimeKeeping_Time); } return(null); case QualifiedSampleWatchSignals.ModeEvt: { LogStateEvent(StateLogType.EventTransition, s_TimeKeeping_Time, s_TimeKeeping_Date, "ModeEvt", "ModeEvt"); TransitionTo(s_TimeKeeping_Date, s_trans_ModeEvt_TimeKeeping_Time_2_TimeKeeping_Date); return(null); } // ModeEvt case QualifiedSampleWatchSignals.TickEvt: { TickTime(); LogStateEvent(StateLogType.EventTransition, s_TimeKeeping_Time, s_TimeKeeping_Time, "TickEvt", "t0-every 1 raise TickEvt/TickTime()"); TransitionTo(s_TimeKeeping_Time, s_trans_t0_TickEvt_TimeKeeping_Time_2_TimeKeeping_Time); return(null); } // TickEvt } // switch return(s_TimeKeeping); } // S_TimeKeeping_Time
protected override QState ActiveTest_Running(IQEvent qEvent) { QState result = base.ActiveTest_Running(qEvent); if (result == null) { return(null); } else { switch (qEvent.QSignal) { case (int)QFSignal.TestStep: NextStep(); return(null); // DESIGN NOTE: Wait for the associated ActiveTest to complete... case (int)QFSignal.TestResult: if (string.IsNullOrEmpty(((TestEvent)qEvent).Reason)) { TransitionToPassed(); } else { TransitionToFailed(((TestEvent)qEvent).Reason); } return(null); } return(result); } }
//Go to a state upon initial transition protected void initial_transition(QState s) { if (state == (QState)superstate(s)) { state = s; } }
public override QState GetRandNextState(QState prevState) { var rng = new Random(); var rand = rng.NextDouble(); Cell result = null; var possibilites = this.GetAvailableStates(); while (result == null) { if (0 <= rand && rand < 0.25 && !NorthWall) { result = Cells[NeighborNorthID]; } if (0.25 <= rand && rand < 0.5 && !SouthWall) { result = Cells[NeighborSouthID]; } if (0.5 <= rand && rand < 0.75 && !WestWall) { result = Cells[NeighborWestID]; } if (0.75 <= rand && rand <= 1 && !EastWall) { result = Cells[NeighborEastID]; } if (prevState != null && result != null && result.Equals((Cell)prevState) && rng.NextDouble() >= 0.6) { result = null; } rand = rng.NextDouble(); } return(result); }
private void ArchiveState(QState state) { if (_history.Count >= MaxHistorySize) { _history.RemoveLast(); } _history.AddFirst(state); }
private string StateNameFrom(QState state) { if (state == null) { return("NULLSTATE"); } return(state.Method.Name); }
} //init private Reminder() { Polling = new QState(this.DoPolling); Processing = new QState(this.DoProcessing); Idle = new QState(this.DoIdle); Busy = new QState(this.DoBusy); Final = new QState(this.DoFinal); } //ctor
private float CalcQValue(QState qState) { float QValue = _wDistToGhost1 * DistanceToGhost(qState.PacmanCoordinates, qState.GhostCoordinates[0]) + _wDistToGhost2 * DistanceToGhost(qState.PacmanCoordinates, qState.GhostCoordinates[1]) + _wDistToFood * DistanceToNearestFood(qState.PacmanCoordinates, qState.FoodCoordinates); return(QValue); }
public BattleBeatenState(GameObject player, GameObject npc) { stateID = StateID.BattleBeaten; CurPlayer = player; CurNpc = npc; anim = npc.GetComponent <Animator>(); MDPQstate = new QState(Action.Beaten); }
/// <summary> /// Создание экземпляра класса <see cref="QChar"/>. /// </summary> public QChar(char chr, QState state, int eulerEquals, int eulerRoundings, int popularity) { State = state; EulerEquals = eulerEquals; EulerRoundings = eulerRoundings; Popularity = popularity; Char = chr; }
// Constructors & Finalizers ------------------------------------------- public ImageFileManageProcess(HSTWorkcell workcell, string processID, string processName) : base(workcell, processID, processName) { // initialize HSM delegates _isNonIdleProcess = true; stateStartRunning = new QState(this.StartRunning); _stateTimer = new QTimer(this); }
protected virtual void QUpdate(int n, QState currentState, QAction action, QState newState, decimal reward) { QStateActionPair p = new QStateActionPair(currentState, action); decimal maxQ = GetMaxValue(newState); QValues[p] = (1 - learn) * GetQValue(p) + learn * (reward + discount * maxQ); UpdateLearningTable(n, currentState, action, QValues[p]); }
protected virtual void CreateStateFields() { if (null == _MyStaticInstance) { s_NoSparks = new QState(S_NoSparks); s_Sparking = new QState(S_Sparking); } }
/// <summary> /// Default constructor - initializes all fields to default values /// </summary> public QHsmTest() { m_s0 = new QState(this.s0); m_s1 = new QState(this.s1); m_s11 = new QState(this.s11); m_s2 = new QState(this.s2); m_s21 = new QState(this.s21); m_s211 = new QState(this.s211); }
public QAgent(QAlgo algo, QState initialState) { trialNum = 0; this.initialState = initialState; initialState.setQAgent(this); currentAlgo = algo; currentAlgo.setQAgent(this); currentAlgo.Initialize(); }
public Philosopher(int philosopherId) { m_PhilosopherId= philosopherId; m_StateThinking = new QState(this.Thinking); m_StateHungry = new QState(this.Hungry); m_StateEating = new QState(this.Eating); m_Timer = new QTimer(this); }
// This runs a single trial/instance of the QState problem. QLearner will automatically run many times for learning or once to apply what has been learned. // Must return the final state public override QState Run(QState currentState, int trialNum, decimal learn, decimal discount, decimal explore) { while (!currentState.IsEnd() && currentState.GetActions().Length > 0 && isRunning) { QAction action = currentState.GetActions().ElementAt(r.Next(currentState.GetActions().Length)); QState newState = currentState.GetNewState(action); newState.Inherit(currentState); newState.Step(); WriteOutput((CurrentMode == LEARN ? "Trial " + trialNum + ", " : "") + ": '" + action + "' @ " + currentState.ToString()); currentState = newState; } return currentState; }
public Table(int numberOfPhilosophers) { m_NumberOfPhilosophers = numberOfPhilosophers; m_ForkIsUsed = new bool[m_NumberOfPhilosophers]; m_PhilosopherIsHungry = new bool[m_NumberOfPhilosophers]; for(int i = 0; i < m_NumberOfPhilosophers; i++) { m_ForkIsUsed[i] = false; m_PhilosopherIsHungry[i] = false; } m_StateServing = new QState(this.Serving); }
private AlarmClock() { TimeKeeping = new QState(this.DoTimeKeeping); Mode12Hour = new QState(this.DoMode12Hour); Mode24Hour = new QState(this.DoMode24Hour); Final = new QState(this.DoFinal); // Create the delegate that invokes methods for the timer. timerDelegate = new TimerCallback(GenerateTimerTickEvent); myAlarm = Alarm.Instance;//we could instead just use "Alarm.Instance" and eliminate "myAlarm" #if USE_DOTNET_EVENTS myAlarm.AlarmActivated += new AlarmClock.AlarmDisplayHandler(AlarmActivated); #endif }
public override QState Run(QState currentState, int trialNum, decimal learn, decimal discount, decimal explore) { this.learn = learn; this.discount = discount; this.explore = explore; decimal score = 0; int actionsTaken = 0; while (!currentState.IsEnd() && GetOutcomes(currentState).Count > 0 && isRunning) { actionsTaken++; QAction a; bool exp; if (explore > 0 && (decimal)random.NextDouble() <= explore) { a = GetRandomAction(currentState); exp = true; } else { a = GetBestAction(currentState); exp = false; } QState newState = currentState.GetNewState(a); WriteOutput((CurrentMode == LEARN ? "Trial " + trialNum + ", " : "") + "#" + actionsTaken + " " + (exp ? "Explore" : "Action") + ": '" + a + "' @ " + currentState.ToString()); newState.Inherit(currentState); newState.Step(); decimal r = GetReward(currentState, newState); score += r; QUpdate(actionsTaken, currentState, a, newState, r); WriteOutput((CurrentMode == LEARN ? "Trial " + trialNum + ", " : "") + "#" + actionsTaken + " Gain " + Math.Round(r, 4) + ", Total " + Math.Round(score, 4)); foreach (KeyValuePair<QStateActionPair, QState> kv in newState.GetObservedStates(currentState, a)) { QState observedPriorState = kv.Key.state; QAction observedAction = kv.Key.action; QState observedState = kv.Value; decimal observedR = GetReward(observedPriorState, observedState); QUpdate(actionsTaken, observedPriorState, observedAction, observedState, observedR); WriteOutput((CurrentMode == LEARN ? "Trial " + trialNum + ", " : "") + "#" + actionsTaken + " Observed: '" + observedAction + "' @ " + observedPriorState.ToString() + " | Gain " + Math.Round(observedR, 4)); } currentState = newState; } if (isRunning) { WriteOutput("Trial " + trialNum + ": " + Math.Round(score, 4) + " in " + actionsTaken + " step" + (actionsTaken == 1 ? "" : "s") + "."); } return currentState; }
// This runs a single trial/instance of the QState problem. QLearner will automatically run many times for learning or once to apply what has been learned. // Must return the final state public virtual QState Run(QState currentState, int trialNum, decimal learn, decimal discount, decimal explore) { /* Should follow the following format: while (!currentState.IsEnd() && currentState.GetActions().Length > 0 && isRunning) { *Pick an action for the current state* ... QState newState = currentState.GetNewState(a); newState.Inherit(currentState); newState.Step(); ... *Learn from the reward/cost of the decision* ... currentState = newState; } * */ return currentState; }
public QSearchResult AStar(QState startState, bool output = false, int maxQueue = 1000) { HashSet<QState> explored = new HashSet<QState>(); Dictionary<QState, decimal> bestSoFar = new Dictionary<QState, decimal>() { { startState, 0 } }; HeapPriorityQueue<QStateContainer> toDo = new HeapPriorityQueue<QStateContainer>(maxQueue); toDo.Enqueue(new QStateContainer(startState), 0); Dictionary<QState, QSearchResult> pathTo = new Dictionary<QState, QSearchResult>() { { startState, new QSearchResult() } }; if (output) WriteOutput("Searching for shortest path via A-Star Search..."); int steps = 0; while (toDo.Count > 0 && isRunning) { steps++; QState current = toDo.Dequeue().qstate; if (current.IsEnd()) { if (output) WriteOutput("Shortest path of " + pathTo[current].Count + " step(s) found after " + steps + " iteration(s)."); return pathTo[current]; } else { explored.Add(current); foreach (QAction action in current.GetActions()) { QState newState = current.GetNewState(action); if (!explored.Contains(newState)) { decimal actualCost = bestSoFar[current] - current.GetValue(); if (!bestSoFar.ContainsKey(newState) || actualCost < bestSoFar[newState]) { pathTo[newState] = new QSearchResult(pathTo[current]); pathTo[newState].actionsList.Add(action); pathTo[newState].QStatesList.Add(newState); bestSoFar[newState] = actualCost; toDo.Enqueue(new QStateContainer(newState), bestSoFar[newState] - 1 * newState.GetValueHeuristic()); } } } } } if (output) WriteOutput("No path found after " + steps + " iteration(s)."); return null; }
public QSearchResult Breadth_First(QState startState, bool output = false) { HashSet<QState> explored = new HashSet<QState>(); Dictionary<QState, decimal> bestSoFar = new Dictionary<QState,decimal>(){{startState, 0}}; Queue<QState> toDo = new Queue<QState>(); toDo.Enqueue(startState); Dictionary<QState, QSearchResult> pathTo = new Dictionary<QState, QSearchResult>() { { startState, new QSearchResult() } }; if(output) WriteOutput("Searching for shortest path via Breadth-First Search..."); int steps = 0; while (toDo.Any() && isRunning) { steps++; QState current = toDo.Dequeue(); if (current.IsEnd()) { if(output) WriteOutput("Shortest path of " + pathTo[current].Count + " step(s) found after " + steps + " iteration(s)."); return pathTo[current]; } else { explored.Add(current); foreach (QAction action in current.GetActions()) { QState newState = current.GetNewState(action); if (!explored.Contains(newState)) { decimal actualCost = bestSoFar[current] - current.GetValue(); if (!bestSoFar.ContainsKey(newState) || actualCost < bestSoFar[newState]) { pathTo[newState] = new QSearchResult(pathTo[current]); pathTo[newState].actionsList.Add(action); pathTo[newState].QStatesList.Add(newState); bestSoFar[newState] = actualCost; toDo.Enqueue(newState); } } } } } if(output) WriteOutput("No path found after " + steps + " iteration(s)."); return null; }
// Update the weights of each feature based on their contribution to the reward protected override void QUpdate(int n, QState currentState, QAction action, QState newState, decimal reward) { decimal maxQ = GetMaxValue(newState); QStateActionPair p = new QStateActionPair(currentState, action); Dictionary<QFeature, decimal> features; if (featureCache.ContainsKey(p)) features = featureCache[p]; else { features = currentState.GetFeatures(action); featureCache[p] = features; } decimal currentQ = GetQValue(p); decimal difference = reward + discount * maxQ - currentQ; foreach (KeyValuePair<QFeature, decimal> feature in features) { try { if (!QWeights.ContainsKey(feature.Key)) QWeights[feature.Key] = 0; decimal oldWeight = QWeights[feature.Key]; decimal newWeight = oldWeight + learn * difference * feature.Value; if (Math.Abs(newWeight) <= 1000000) { QWeights[feature.Key] = newWeight; } else WriteOutput("Warning: Weights diverging. Check that your features are valid and measured consistently with everything else.", true); } catch (Exception e) { WriteOutput("Exception: " + e + "\n*Check that your features are valid and measured consistently with everything else.*", true); Abort(); break; } } // Output foreach (QFeature f in features.Keys) { UpdateLearningTable(-1, f.ToString(), QWeights[f].ToString(), features[f]); } }
protected virtual void RaiseFinalStateReached(EventHandler handler, ILQHsm hsm, QState state) { if (handler != null) { LogStateEventArgs logEvent = new LogStateEventArgs (); logEvent.LogType = StateLogType.Final; logEvent.State = state; handler (hsm, logEvent); } }
public override void Open(object o, QState initialState) { QWeights = (Dictionary<QFeature, decimal>)o; ClearLearningTable(); int i = 1; foreach (KeyValuePair<QFeature, decimal> kv in QWeights) UpdateLearningTable(i++, kv.Key.ToString(), kv.Value.ToString(), 0); }
/// <summary> /// 添加状态 /// </summary> /// <param name="state">State.</param> public void AddState(string name) { StateDict [name] = new QState(name); }
public override void Open(object o, QState initialState) { QValues.Clear(); ClearLearningTable(); foreach (KeyValuePair<object[], decimal> kv in ((Dictionary<object[], decimal>)o)) { QValues.Add(new QStateActionPair(initialState.Open(kv.Key[0]), (QAction)kv.Key[1]), kv.Value); } int i=1; foreach (KeyValuePair<QStateActionPair, decimal> kv in QValues) { UpdateLearningTable(i++, kv.Key.state, kv.Key.action, kv.Value); } }
public QAgent(QLearner master, QAlgo algo, QState initialState) : this(algo, initialState) { this.master = master; }
// Return a random action for exploration protected virtual QAction GetRandomAction(QState state) { IEnumerable<QAction> s = GetOutcomes(state).Select(x => x.action); return s.ElementAt(random.Next(s.Count())); }
/// <summary> /// Default constructor - initializes all fields to default values /// </summary> public QHsmBase1() { m_s0 = new QState(this.s0); m_s01 = new QState(this.s01); m_s02 = new QState(this.s02); }
// Return best action to take from current state (best QValue or random if tied) protected virtual QAction GetBestAction(QState state) { decimal maxVal = GetMaxValue(state); IEnumerable<QAction> s = GetOutcomes(state).Where(x => GetQValue(x) == maxVal).Select(x => x.action); int n = s.Count(); if (n == 1) return s.First(); else { return s.ElementAt(random.Next(n)); } }
// Return the value of the best action taken at given state, else 0 if not known protected virtual decimal GetMaxValue(QState state) { return GetOutcomes(state).Select(x => GetQValue(x)).Max(); }
// Return list of all possible states that can result from an action taken from the current state protected virtual List<QStateActionPair> GetOutcomes(QState state) { return state.GetActions().Select(x => new QStateActionPair(state, x)).ToList() ; }
public virtual decimal GetReward(QState currentState, QState newState) { return newState.GetValue() - currentState.GetValue(); }
private void LoadQStatePlugin(string f) { try { bool loaded = false; foreach(Assembly a in AppDomain.CurrentDomain.GetAssemblies()) foreach (Type qstate in a.GetTypes(). Where(t => String.Equals(t.Namespace, "QLearner.QStates", StringComparison.Ordinal)). Where(t => t.IsSubclassOf(typeof(QState))). Where(t => t.IsVisible). Where(t => !t.IsAbstract). Where(t => t.Name==f)) { state = (QState)Activator.CreateInstance(qstate); WriteOutput("Loaded QState: " + qstate.Name, true); state.setQLearner(this); Settings.Enabled = state.HasSettings; if (state != null && algo != null) { agent = new QAgent(this, algo, state); Reset(); Awaken.Enabled = Learn.Enabled = true; } Properties.Settings.Default.QStatePlugin = qstate.Name; Properties.Settings.Default.Save(); loaded = true; break; } if (!loaded) { WriteOutput("QState Not Found: " + f, true); ResetPlugins(false); } } catch (Exception e) { WriteOutput("Unable to load QState: " + f +"\n"+e, true); ResetPlugins(false); } }
public void UpdateLearningTable(int n, QState s, QAction a, decimal qv) { if (InvokeRequired) { Invoke(new UpdateLearningTableD(UpdateLearningTable), n, s, a, qv); } else { QStateActionPair p = new QStateActionPair(s, a); if (LearningTableQStateKeys.ContainsKey(p)) { LearningTableQStateKeys[p].Cells["Num"].Value = n; LearningTableQStateKeys[p].Cells["QValue"].Value = qv; } else { LearningTable.Rows.Add(n, s.ToString(), a.ToString(), qv); LearningTableQStateKeys[p] = LearningTable.Rows[LearningTable.Rows.Count - 1]; ; } } }
public QStateActionPair(QState s, QAction a) { state = s; action = a; }
private QState SlowTwo; //step 2 -- part 2 of long op that user may want to cancel #endregion Fields #region Constructors private RunToCompletion() { Dispatching = new QState(this.DoDispatching); CantInterrupt = new QState(this.DoCantInterrupt); Interruptible = new QState(this.DoInterruptible); SlowOne = new QState(this.DoSlowOne); SlowTwo = new QState(this.DoSlowTwo); Completed = new QState(this.DoCompleted); Final = new QState(this.DoFinal); }
/// <summary> /// Default constructor - initializes all fields to default values /// </summary> public QHsmDerived3() { m_s021 = new QState(this.s021); }
// The below functions are called to open and save files holding what has been learned. QLearner will handle the file write/read operations and receive or pass and general object to the QAlgo. Make sure what you pass in Save is a serializable object. If what you save includes QState, use the Open from the initialState to rebuild the state. public virtual void Open(object o, QState initialState) { }