/*private void AddToLongTermMemory(VirusMemory memory, double significance) {
if (LongTermMemory.Count < longTermMemorySize) {
for (int i = 0; i <= LongTermMemory.Count; i++) {
if (i == LongTermMemory.Count) {
LongTermMemory.Add(new VirusLongTermMemory(memory, significance));
break;
}
else {
if (LongTermMemory[i].Significance < significance) {
LongTermMemory.Insert(i, new VirusLongTermMemory(memory, significance));
break;
}
}
}
}
else {
if (LongTermMemory[LongTermMemory.Count - 1].Significance > significance)
return;
for (int i = 0; i < LongTermMemory.Count; i++) {
if (LongTermMemory[i].Significance <= significance) {
LongTermMemory.Insert(i, new VirusLongTermMemory(memory, significance));
LongTermMemory.RemoveAt(LongTermMemory.Count - 1);
break;
}
}
}
}*/
private void AddToLongTermMemory(VirusMemory[] episode, double significance) {
if (LongTermMemory.Count < longTermMemorySize) {
for (int i = 0; i <= LongTermMemory.Count; i++) {
if (i == LongTermMemory.Count) {
LongTermMemory.Add(new VirusMemoryEpisode(episode, significance));
break;
}
else {
if (LongTermMemory[i].Significance < significance) {
LongTermMemory.Insert(i, new VirusMemoryEpisode(episode, significance));
break;
}
}
}
}
else {
if (LongTermMemory[LongTermMemory.Count - 1].Significance > significance)
return;
for (int i = 0; i < LongTermMemory.Count; i++) {
if (LongTermMemory[i].Significance <= significance) {
LongTermMemory.Insert(i, new VirusMemoryEpisode(episode, significance));
LongTermMemory.RemoveAt(LongTermMemory.Count - 1);
break;
}
}
}
}
private double Learn(VirusMemory memory) {
VirusBoard startstate = memory.StartState;
VirusBoard endstate = memory.EndState;
Move action = memory.Action;
double reward = memory.Reward;
// -- Make sure the entries for the state and action exist --
if (!Q.ContainsKey(startstate.CustomHash()))
Q.Add(startstate.CustomHash(), new Dictionary<UInt32, double>());
if (!Q[startstate.CustomHash()].ContainsKey(action.CustomHash()))
Q[startstate.CustomHash()].Add(action.CustomHash(), initvalue);
if (!N.ContainsKey(startstate.CustomHash()))
N.Add(startstate.CustomHash(), new Dictionary<UInt32, int>());
if (!N[startstate.CustomHash()].ContainsKey(action.CustomHash()))
N[startstate.CustomHash()].Add(action.CustomHash(), 0);
// -- Perform the update of Q-values --
N[startstate.CustomHash()][action.CustomHash()]++;
double change = LearningRate(N[startstate.CustomHash()][action.CustomHash()])
* (reward + discount * GetMaxQ(endstate) - Q[startstate.CustomHash()][action.CustomHash()]);
Q[startstate.CustomHash()][action.CustomHash()] =
Q[startstate.CustomHash()][action.CustomHash()] + change;
return change;
}
public int TellOfMemoryToExt(MemoryQAgent agent, bool fullepisode = false)
{
VirusMemory[] memories = LongTermMemory[random.Next(LongTermMemory.Count)].Memories;
VirusMemory memory = memories[0];
if (fullepisode)
{
agent.LearnFromEpisode(memories, true);
//foreach (VirusMemory m in memories)
// agent.Learn(m);
return(memories.Length);
}
else
{
foreach (VirusMemory m in memories)
{
if (Math.Abs(m.Reward) > Math.Abs(memory.Reward))
{
memory = m;
}
}
agent.Learn(memory);
return(1);
}
}
private double Learn(VirusMemory memory)
{
VirusBoard startstate = memory.StartState;
VirusBoard endstate = memory.EndState;
Move action = memory.Action;
double reward = memory.Reward;
// -- Make sure the entries for the state and action exist --
if (!Q.ContainsKey(startstate.CustomHash()))
{
Q.Add(startstate.CustomHash(), new Dictionary <UInt32, double>());
}
if (!Q[startstate.CustomHash()].ContainsKey(action.CustomHash()))
{
Q[startstate.CustomHash()].Add(action.CustomHash(), initvalue);
}
if (!N.ContainsKey(startstate.CustomHash()))
{
N.Add(startstate.CustomHash(), new Dictionary <UInt32, int>());
}
if (!N[startstate.CustomHash()].ContainsKey(action.CustomHash()))
{
N[startstate.CustomHash()].Add(action.CustomHash(), 0);
}
// -- Perform the update of Q-values --
N[startstate.CustomHash()][action.CustomHash()]++;
double change = LearningRate(N[startstate.CustomHash()][action.CustomHash()])
* (reward + discount * GetMaxQ(endstate) - Q[startstate.CustomHash()][action.CustomHash()]);
Q[startstate.CustomHash()][action.CustomHash()] =
Q[startstate.CustomHash()][action.CustomHash()] + change;
return(change);
}
private double LearnFromEpisode(VirusMemory[] episode, bool reverse = false) {
double significance = 0;
for (int i = 0; i < episode.Length; i++) {
int index = reverse ? episode.Length - i - 1 : i;
double change = Learn(episode[index]);
significance += (change < 0 ? Math.Abs(change) : change*4);
}
//foreach (VirusMemory memory in episode) {
// significance += Learn(memory);
//}
return significance/episode.Length;
}
private VirusMemory[][] ExtractEpisodesFromMemory()
{
List <VirusMemory[]> episodes = new List <VirusMemory[]>();
List <VirusMemory> episode = new List <VirusMemory>();
int n = 0;
for (int i = 0; i < ShortTermMemory.Count; i++)
{
VirusMemory memory = ShortTermMemory[i];
byte winner = memory.EndState.winner;
episode.Add(memory);
if (winner != 0)
{
episodes.Add(episode.ToArray());
episode.Clear();
n = i;
}
}
ShortTermMemory.RemoveRange(0, n + 1);
return(episodes.ToArray());
}
public void TellOfMemoryTo(MemoryQAgent agent, bool fullepisode = false)
{
VirusMemory[] memories = LongTermMemory[random.Next(LongTermMemory.Count)].Memories;
VirusMemory memory = memories[0];
if (fullepisode)
{
foreach (VirusMemory m in memories)
{
agent.Learn(m);
}
}
else
{
foreach (VirusMemory m in memories)
{
if (Math.Abs(m.Reward) > Math.Abs(memory.Reward))
{
memory = m;
}
}
agent.Learn(memory);
}
}
