/*private void AddToLongTermMemory(VirusMemory memory, double significance) { if (LongTermMemory.Count < longTermMemorySize) { for (int i = 0; i <= LongTermMemory.Count; i++) { if (i == LongTermMemory.Count) { LongTermMemory.Add(new VirusLongTermMemory(memory, significance)); break; } else { if (LongTermMemory[i].Significance < significance) { LongTermMemory.Insert(i, new VirusLongTermMemory(memory, significance)); break; } } } } else { if (LongTermMemory[LongTermMemory.Count - 1].Significance > significance) return; for (int i = 0; i < LongTermMemory.Count; i++) { if (LongTermMemory[i].Significance <= significance) { LongTermMemory.Insert(i, new VirusLongTermMemory(memory, significance)); LongTermMemory.RemoveAt(LongTermMemory.Count - 1); break; } } } }*/ private void AddToLongTermMemory(VirusMemory[] episode, double significance) { if (LongTermMemory.Count < longTermMemorySize) { for (int i = 0; i <= LongTermMemory.Count; i++) { if (i == LongTermMemory.Count) { LongTermMemory.Add(new VirusMemoryEpisode(episode, significance)); break; } else { if (LongTermMemory[i].Significance < significance) { LongTermMemory.Insert(i, new VirusMemoryEpisode(episode, significance)); break; } } } } else { if (LongTermMemory[LongTermMemory.Count - 1].Significance > significance) return; for (int i = 0; i < LongTermMemory.Count; i++) { if (LongTermMemory[i].Significance <= significance) { LongTermMemory.Insert(i, new VirusMemoryEpisode(episode, significance)); LongTermMemory.RemoveAt(LongTermMemory.Count - 1); break; } } } }
private double Learn(VirusMemory memory) { VirusBoard startstate = memory.StartState; VirusBoard endstate = memory.EndState; Move action = memory.Action; double reward = memory.Reward; // -- Make sure the entries for the state and action exist -- if (!Q.ContainsKey(startstate.CustomHash())) Q.Add(startstate.CustomHash(), new Dictionary<UInt32, double>()); if (!Q[startstate.CustomHash()].ContainsKey(action.CustomHash())) Q[startstate.CustomHash()].Add(action.CustomHash(), initvalue); if (!N.ContainsKey(startstate.CustomHash())) N.Add(startstate.CustomHash(), new Dictionary<UInt32, int>()); if (!N[startstate.CustomHash()].ContainsKey(action.CustomHash())) N[startstate.CustomHash()].Add(action.CustomHash(), 0); // -- Perform the update of Q-values -- N[startstate.CustomHash()][action.CustomHash()]++; double change = LearningRate(N[startstate.CustomHash()][action.CustomHash()]) * (reward + discount * GetMaxQ(endstate) - Q[startstate.CustomHash()][action.CustomHash()]); Q[startstate.CustomHash()][action.CustomHash()] = Q[startstate.CustomHash()][action.CustomHash()] + change; return change; }
public int TellOfMemoryToExt(MemoryQAgent agent, bool fullepisode = false) { VirusMemory[] memories = LongTermMemory[random.Next(LongTermMemory.Count)].Memories; VirusMemory memory = memories[0]; if (fullepisode) { agent.LearnFromEpisode(memories, true); //foreach (VirusMemory m in memories) // agent.Learn(m); return(memories.Length); } else { foreach (VirusMemory m in memories) { if (Math.Abs(m.Reward) > Math.Abs(memory.Reward)) { memory = m; } } agent.Learn(memory); return(1); } }
private double Learn(VirusMemory memory) { VirusBoard startstate = memory.StartState; VirusBoard endstate = memory.EndState; Move action = memory.Action; double reward = memory.Reward; // -- Make sure the entries for the state and action exist -- if (!Q.ContainsKey(startstate.CustomHash())) { Q.Add(startstate.CustomHash(), new Dictionary <UInt32, double>()); } if (!Q[startstate.CustomHash()].ContainsKey(action.CustomHash())) { Q[startstate.CustomHash()].Add(action.CustomHash(), initvalue); } if (!N.ContainsKey(startstate.CustomHash())) { N.Add(startstate.CustomHash(), new Dictionary <UInt32, int>()); } if (!N[startstate.CustomHash()].ContainsKey(action.CustomHash())) { N[startstate.CustomHash()].Add(action.CustomHash(), 0); } // -- Perform the update of Q-values -- N[startstate.CustomHash()][action.CustomHash()]++; double change = LearningRate(N[startstate.CustomHash()][action.CustomHash()]) * (reward + discount * GetMaxQ(endstate) - Q[startstate.CustomHash()][action.CustomHash()]); Q[startstate.CustomHash()][action.CustomHash()] = Q[startstate.CustomHash()][action.CustomHash()] + change; return(change); }
private double LearnFromEpisode(VirusMemory[] episode, bool reverse = false) { double significance = 0; for (int i = 0; i < episode.Length; i++) { int index = reverse ? episode.Length - i - 1 : i; double change = Learn(episode[index]); significance += (change < 0 ? Math.Abs(change) : change*4); } //foreach (VirusMemory memory in episode) { // significance += Learn(memory); //} return significance/episode.Length; }
private VirusMemory[][] ExtractEpisodesFromMemory() { List <VirusMemory[]> episodes = new List <VirusMemory[]>(); List <VirusMemory> episode = new List <VirusMemory>(); int n = 0; for (int i = 0; i < ShortTermMemory.Count; i++) { VirusMemory memory = ShortTermMemory[i]; byte winner = memory.EndState.winner; episode.Add(memory); if (winner != 0) { episodes.Add(episode.ToArray()); episode.Clear(); n = i; } } ShortTermMemory.RemoveRange(0, n + 1); return(episodes.ToArray()); }
public void TellOfMemoryTo(MemoryQAgent agent, bool fullepisode = false) { VirusMemory[] memories = LongTermMemory[random.Next(LongTermMemory.Count)].Memories; VirusMemory memory = memories[0]; if (fullepisode) { foreach (VirusMemory m in memories) { agent.Learn(m); } } else { foreach (VirusMemory m in memories) { if (Math.Abs(m.Reward) > Math.Abs(memory.Reward)) { memory = m; } } agent.Learn(memory); } }