public FoxItemActor()
{
List <FoxNewsItem> items = new List <FoxNewsItem>();
var counterActor = Context.ActorSelection("akka://feed-system/user/wordCount");
Receive <FoxNews.processFoxItem>(i =>
{
var id = Helpers.GeneralHelper.IdHelper(i.item.Element("guid").Value);
id = Helpers.GeneralHelper.IdHelper(id);
if (!idsProcessed.Contains(id))
{
idsProcessed.Add(id);
var d = i.item.Element("description").Value;
d = d.DeEscape();
if (d.IndexOf("<") != 0)
{
var item = new FoxNewsItem()
{
id = i.item.Element("guid").Value.Replace("/", ""),
partionKey = "FOX",
link = i.item.Element("link").Value,
pubDate = DateTimeOffset.Parse(i.item.Element("pubDate").Value),
title = i.item.Element("title").Value.DeEscape(),
description = d.IndexOf("<") == -1? d : d.Substring(0, d.IndexOf("<") - 1),
origXML = i.item.ToString(),
};
var idx = item.link.IndexOf("/", 24);
item.siteSection = item.link.Substring(24, idx - 24);
items.Add(item);
Program.stopActor.Tell(new NLPHelper.StopwordRequest()
{
id = item.id,
linesOfText = new List <string>()
{
item.title, item.description
}
});
}
else
{
Context.Parent.Tell(new processedFox());
}
}
else
{
Context.Parent.Tell(new processedFox());
}
});
Receive <NLPHelper.StopwordResponse>(r =>
{
var item = items.First(z => z.id == r.id);
item.swTitle = r.outputStrings[0];
item.swDescription = r.outputStrings[1];
var req = new StemmingRequest()
{
id = item.id,
linesOfText = new List <string>()
{
item.swTitle,
item.swDescription,
}
};
Program.stemActor.Tell(req);
});
Receive <NLPHelper.StemmingResponse>(r =>
{
var item = items.First(z => z.id == r.id);
item.stemmedTitle = r.lines[0];
item.stemmedDescription = r.lines[1];
var n = new SharedMessages.SentimentRequest()
{
id = r.id,
feed = "FOX",
linesToProcess = new List <string>()
{
item.title,
item.description
}
};
var remote2 = Context.ActorSelection("akka.tcp://nlp-system@localhost:8080/user/sent");
var req2 = JsonConvert.SerializeObject(n);
remote2.Tell(req2);
});
// step 3 || step 4
Receive <string>(r =>
{
if (r.StartsWith("sent:"))
{
var sent = JsonConvert.DeserializeObject <SharedMessages.SentimentResponse>(r.Substring(5));
var item = items.First(z => z.id == sent.id);
if (string.IsNullOrWhiteSpace(item.title))
{
item.sentiDescription = sent.results[0];
}
else if (string.IsNullOrWhiteSpace(item.description))
{
item.sentTitle = sent.results[0];
}
else
{
item.sentTitle = sent.results[0];
item.sentiDescription = sent.results[1];
}
var n = new SharedMessages.NERRequest()
{
id = item.id,
feed = "FOX",
linesToProcess = new List <string>()
{
item.title,
item.description
}
};
var remote2 = Context.ActorSelection("akka.tcp://nlp-system@localhost:8080/user/ner");
var req2 = JsonConvert.SerializeObject(n);
remote2.Tell(req2);
}
else if (r.StartsWith("ner:"))
{
var sent = JsonConvert.DeserializeObject <SharedMessages.NERResponse>(r.Substring(4));
var item = items.First(z => z.id == sent.id);
item.nerTitle = sent.results[0];
item.nerDescription = sent.results[1];
Program.nerCountActor.Tell(sent);
Self.Tell(new processedFox()
{
id = item.id
});
}
});
Receive <processedFox>(r =>
{
var item = items.First(z => z.id == r.id);
Program.cdb.UpsertDocument(item, "newsfeed").Wait();
items.Remove(item);
Console.WriteLine("=========> Fox saving " + item.siteSection);
counterActor.Tell(new NLPHelper.CountRequest()
{
Feed = "Fox",
Id = item.id,
LineOFtext = item.stemmedDescription
});
Program.nerActor.Tell(new SharedMessages.NERRequest()
{
id = item.id,
feed = "FOX",
linesToProcess = new List <string>()
{
item.title,
item.description
}
});
Context.Parent.Tell(new processedFox());
});
}
public CNNItemActor()
{
List <CNNItem> items = new List <CNNItem>();
Receive <processCNNItem>(i =>
{
if (i.item.Element("guid") != null)
{
var id = Helpers.GeneralHelper.IdHelper(i.item.Element("guid").Value);
id = Helpers.GeneralHelper.IdHelper(id);
if (!idsProcessed.Contains(id))
{
idsProcessed.Add(id);
var d = i.item.Element("description").Value;
d = d.DeEscape();
if (d.IndexOf("<") > 0 && !d.StartsWith("http"))
{
DateTimeOffset dt = DateTimeOffset.Now;
// "Sat, 30 Mar 2019 14:26:25 -0400"
if (i.item.Element("pubDate") != null)
{
var pub = i.item.Element("pubDate").Value;
pub = pub.Substring(0, pub.Length - 4) + " +0000";
dt = DateTimeOffset.Parse(pub);
}
var item = new CNNItem()
{
id = id,
partionKey = "CNN",
link = i.item.Element("link").Value,
pubDate = dt,
title = i.item.Element("title").Value.DeEscape(),
description = d.Substring(0, d.IndexOf("<") - 1),
origXML = i.item.ToString(),
};
var g = i.item.Element("guid").Value;
if (g.Length > 35)
{
var idx = g.IndexOf("/", 31);
item.siteSection = g.Substring(31, g.IndexOf("/", 31) - 31);
}
else
{
item.siteSection = g;
}
items.Add(item);
Program.stopActor.Tell(new NLPHelper.StopwordRequest()
{
id = item.id,
linesOfText = new List <string>()
{
item.title, item.description
}
});
}
else
{
Context.Parent.Tell(new processedCNN());
}
}
else
{
Context.Parent.Tell(new processedCNN());
}
}
else
{
Context.Parent.Tell(new processedCNN());
}
});
//Step 2
Receive <NLPHelper.StopwordResponse>(r =>
{
var item = items.First(z => z.id == r.id);
item.swTitle = r.outputStrings[0];
item.swDescription = r.outputStrings[1];
var req = new StemmingRequest()
{
id = item.id,
linesOfText = new List <string>()
{
item.swTitle,
item.swDescription,
}
};
Program.stemActor.Tell(req);
});
// Step 3
Receive <NLPHelper.StemmingResponse>(r =>
{
var item = items.First(z => z.id == r.id);
item.stemmedTitle = r.lines[0];
item.stemmedDescription = r.lines[1];
var n = new SharedMessages.SentimentRequest()
{
id = r.id,
feed = "CNN",
linesToProcess = new List <string>()
{
item.title,
item.description
}
};
var remote = Context.ActorSelection("akka.tcp://nlp-system@localhost:8080/user/sent");
var req = JsonConvert.SerializeObject(n);
remote.Tell(req);
});
// Step 4 = sentiment || Step 5 = ner
Receive <string>(r =>
{
if (r.StartsWith("sent:"))
{
var sent = JsonConvert.DeserializeObject <SharedMessages.SentimentResponse>(r.Substring(5));
var item = items.First(z => z.id == sent.id);
item.sentTitle = sent.results[0];
item.sentDescription = sent.results[1];
var n = new SharedMessages.NERRequest()
{
id = sent.id,
feed = "CNN",
linesToProcess = new List <string>()
{
item.title,
item.description
}
};
var remote = Context.ActorSelection("akka.tcp://nlp-system@localhost:8080/user/ner");
var req = JsonConvert.SerializeObject(n);
remote.Tell(req);
}
else if (r.StartsWith("ner:"))
{
var sent = JsonConvert.DeserializeObject <SharedMessages.NERResponse>(r.Substring(4));
var item = items.First(z => z.id == sent.id);
item.nerTitle = sent.results[0];
item.nerDescription = sent.results[1];
Program.nerCountActor.Tell(sent);
Self.Tell(new processedCNN()
{
id = sent.id
});
}
});
// done!
Receive <processedCNN>(r =>
{
var item = items.First(z => z.id == r.id);
Program.cdb.UpsertDocument(item, "newsfeed").Wait();
items.Remove(item);
Console.WriteLine("=========> CNN saving " + item.siteSection);
Program.wordCountActor.Tell(new NLPHelper.CountRequest()
{
Feed = "CNN",
Id = item.id,
LineOFtext = item.stemmedDescription
});
Program.nerActor.Tell(new SharedMessages.NERRequest()
{
id = item.id,
feed = "CNN",
linesToProcess = new List <string>()
{
item.title,
item.description
}
});
Context.Parent.Tell(new processedCNN());
});
}