void AQSubscriberIdentitySnapshot_OnMessageReceived(string message)
{
try {
lock (_mqLock) {
StoreEventData(_activeMQSettings.TopicNameIdentitySnapshot, message);
IncreaseProcessedCount();
HtmlAgilityPack.XmlDocument xmlDoc = new HtmlAgilityPack.XmlDocument();
xmlDoc.LoadXml(message);
_keui.IdentitySnapshot(xmlDoc);
}
}
catch (Exception ex) {
AddEvent("Exception while processing custom item to index: " + ex.Message);
GenLib.Log.LogService.LogException("Exception while processing custom item to index. message: " + message, ex);
}
}
void AQSubscriberIssueUpdate_OnMessageReceived(string message)
{
try
{
lock (_mqLock)
{
StoreEventData(_activeMQSettings.TopicNameIssueUpdate, message);
IncreaseProcessedCount();
HtmlAgilityPack.XmlDocument xmlDoc = new HtmlAgilityPack.XmlDocument();
xmlDoc.LoadXml(message);
_keui.AnnotateIssueUpdate(xmlDoc);
bool ok = _keui.IndexIssueUpdate(xmlDoc);
if (!ok)
StoreEventData(_activeMQSettings.TopicNameIssueUpdate, message, true, "invalid_");
UpdateEventMetaData(xmlDoc, _activeMQSettings.TopicNameKEUIPublishBugComment);
// publish it
string annotatedMessage = xmlDoc.DocumentNode.OuterHtml;
StoreEventData(_activeMQSettings.TopicNameKEUIPublishBugComment, annotatedMessage);
using (var publisher = new ActiveMqHelper.TopicPublisher(AQSession, _activeMQSettings.TopicNameKEUIPublishBugComment))
{
publisher.SendMessage(annotatedMessage);
}
}
}
catch (Exception ex)
{
AddEvent("Exception while annotating new bug comment: " + ex.Message);
GenLib.Log.LogService.LogException("Exception while processing new bug comment. message: " + message, ex);
}
}
public void LoadSettings()
{
//string currentFolder = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location);
string currentFolder = SettingsServer.AppFolder;
AddEvent("Loading settings from folder " + currentFolder);
try
{
if (!Directory.Exists(Defaults.TemplateFolder))
AddEvent(String.Format("The folder with templates ({0}) does not exist. Create it and put templates there.", Defaults.TemplateFolder));
_activeMQSettings = new ActiveMQSettings(Path.Combine(currentFolder, _fileNameActiveMQSettings));
if (!string.IsNullOrEmpty(_activeMQSettings.LastInfo))
AddEvent("Error while loading Active MQ settings: " + _activeMQSettings.LastInfo);
else
AddEvent("Active MQ settings loaded successfully.");
if (File.Exists(Path.Combine(currentFolder, _fileNameSettings)))
{
string text = File.ReadAllText(Path.Combine(currentFolder, _fileNameSettings));
HtmlAgilityPack.XmlDocument xmlDoc = new HtmlAgilityPack.XmlDocument();
xmlDoc.LoadXml(text);
var indexingNode = xmlDoc.DocumentNode.SelectSingleNode("/settings/indexingService");
if (indexingNode != null)
_defaultProfileName = indexingNode.GetAttributeValue("profileName", "ProfileData");
var pathNode = xmlDoc.DocumentNode.SelectSingleNode("/settings/paths");
if (pathNode != null)
{
_keui.IgnoredConceptsFileName = pathNode.GetAttributeValue("ignoredConcepts", _keui.IgnoredConceptsFileName);
if (!_keui.IgnoredConceptsFileName.Contains(Path.DirectorySeparatorChar))
_keui.IgnoredConceptsFileName = Path.Combine(currentFolder, _keui.IgnoredConceptsFileName);
_keui.CustomLemmasFileName = pathNode.GetAttributeValue("customLemmas", _keui.CustomLemmasFileName);
if (!_keui.CustomLemmasFileName.Contains(Path.DirectorySeparatorChar))
_keui.CustomLemmasFileName = Path.Combine(currentFolder, _keui.CustomLemmasFileName);
}
}
_keui.LoadSettings(currentFolder);
}
catch (Exception ex)
{
GenLib.Log.LogService.LogException("KEUI.LoadSettings failed.", ex);
AddEvent("KEUI.SaveSettings failed." + ex.Message);
}
}
void AQSubscriberConceptNew_OnMessageReceived(string message)
{
try
{
lock (_mqLock)
{
AddEvent("Adding new rdf data to the annotation ontology. " + GetDateStamp());
StoreEventData(_activeMQSettings.TopicNameConceptNew, message);
HtmlAgilityPack.XmlDocument xmlDoc = new HtmlAgilityPack.XmlDocument();
xmlDoc.LoadXml(message);
var newRDFDataNode = xmlDoc.DocumentNode.SelectSingleNode("//ns1:eventData/ns1:newRDFData");
if (newRDFDataNode == null) { AddEventAndLog("The event did not contain //ns1:eventData/s1:newRDFData node. Ignoring event"); return; }
string rdfData = newRDFDataNode.InnerHtml;
rdfData = rdfData.Trim(); // make sure the <xml > tag is in the first line
if (!rdfData.StartsWith("<?xml version=\"1.0\"?>"))
rdfData = "<?xml version=\"1.0\"?>" + Environment.NewLine + rdfData;
_keui.AddNewRDFData(rdfData);
// todo: maybe also use the new concepts to tag existing data
}
}
catch (Exception ex)
{
AddEvent("Exception while processing new RDF data: " + ex.Message);
GenLib.Log.LogService.LogException("Exception while processing new RDF data. message: " + message, ex);
}
}
public string AddPeopleInfoToResults(string returnData)
{
try {
if (string.IsNullOrEmpty(returnData))
return returnData;
HtmlAgilityPack.XmlDocument returnDoc = new HtmlAgilityPack.XmlDocument();
returnDoc.LoadXml(returnData);
char[] minusArray = "-".ToCharArray();
char[] commaArray = ",".ToCharArray();
HashSet<int> detectedAccountIds = new HashSet<int>();
string[] roles = ProfileSettings.PeopleRoles.Split(commaArray, StringSplitOptions.RemoveEmptyEntries);
foreach (HtmlAgilityPack.HtmlNode itemNode in returnDoc.DocumentNode.SelectNodes("//item") ?? new HtmlNodeCollection(null)) {
foreach (string role in roles) {
string[] ids = itemNode.GetAttributeValue(role, "").Split(commaArray, StringSplitOptions.RemoveEmptyEntries);
foreach (string id in ids)
detectedAccountIds.Add(int.Parse(id));
}
}
var peopleNode = returnDoc.DocumentNode.SelectSingleNode("/results/fromToCounts");
if (peopleNode != null) {
string[] items = peopleNode.InnerText.RemoveWhiteSpaces().Split(commaArray, StringSplitOptions.RemoveEmptyEntries);
foreach (string item in items) {
string[] parts = item.Split(minusArray, StringSplitOptions.RemoveEmptyEntries);
detectedAccountIds.Add(int.Parse(parts[0]));
detectedAccountIds.Add(int.Parse(parts[1]));
}
}
if (detectedAccountIds.Count == 0)
return returnData;
StringBuilder peopleInfo = new StringBuilder();
peopleInfo.Append("\t<peopleData>\n");
foreach (int id in detectedAccountIds) {
IPersonInfo p = PeopleData.GetPersonFromAccount(id);
peopleInfo.Append(String.Format("\t\t<person id=\"{0}\" account=\"{1}\" name=\"{2}\" uuid=\"{3}\" />\n", id, PeopleData.GetAccount(id).EncodeXMLString(), p.Name.EncodeXMLString(), p.CustomData.EncodeXMLString()));
}
peopleInfo.Append("\t</peopleData>\n");
Match resultsMatch = Regex.Match(returnData, @"<results[^>]*>\n");
Trace.Assert(resultsMatch.Success == true, "Unable to find <results> section in the result data");
int insertIndex = resultsMatch.Index + resultsMatch.Length;
return returnData.Insert(insertIndex, peopleInfo.ToString());
}
catch (Exception ex) {
GenLib.Log.LogService.LogException("MailData. AddPeopleInfoToResults: ", ex);
}
return "";
}
public string ExecuteCommand(string commandData)
{
string ret = "";
if (string.IsNullOrEmpty(commandData))
return ret;
DateTime start = DateTime.Now;
MailData.LogInfo("ExecuteCommand command: " + commandData.Trim());
HtmlAgilityPack.XmlDocument commandDoc = new HtmlAgilityPack.XmlDocument();
commandDoc.LoadXml(commandData);
var commandNode = commandDoc.DocumentNode.SelectSingleNode("/commandData");
string target = commandNode.GetAttributeValue("target", null);
string command = commandNode.GetAttributeValue("command", null);
if (target == "PeopleData")
ret = PeopleData.ExecuteCommand(command, commandNode);
else if (target == "MailData")
ret = MailData.ExecuteCommand(command, commandNode);
else if (target == "Miner") {
ret = MailData.MinerExecuteCommand(commandData);
GenLib.Log.LogService.LogInfo("Profile. ExecuteCommand result: " + ret);
}
else
MailData.LogInfo("Warning: Don't know how to process the command: " + commandData);
MailData.LogInfo(String.Format("ExecuteCommand needed {0:N0} ms, returning {1:N0} bytes", (DateTime.Now - start).TotalMilliseconds, ret.Length));
return ret;
}
public string ProcessKEUIRequest(HtmlAgilityPack.XmlDocument xmlDoc, string requestType)
{
Debug.Assert(MailData != null, "MailData is null. The indexing service has to be initialize before requests can be processed.");
var requestNode = xmlDoc.DocumentNode.SelectSingleNode("//ns1:eventData/s1:keuiRequest");
string requestData = requestNode.SelectSingleNode("./s1:requestData").InnerHtml;
//requestData = requestData.Replace("<s1:", "<").Replace("</s1:", "</");
var sequenceNumberNode = xmlDoc.DocumentNode.SelectSingleNode("//ns1:head/ns1:sequencenumber");
if (sequenceNumberNode != null)
Trace.WriteLine("Received request with sequencenumber: " + sequenceNumberNode.InnerText);
string returnData = "";
if (requestType == "Query") {
requestData = ChangeBugIdToThreadId(requestData); // if we have specified a bug id this will fix it to threadId
requestData = FixQueryArgs(requestData);
returnData = ProcessQuery(requestData);
returnData = returnData.Replace("<?xml version=\"1.0\" encoding=\"utf-8\"?>", "");
}
else if (requestType == "GetSimilarConcepts") {
requestData = FixQueryArgs(requestData);
returnData = ProcessQuery(requestData);
HtmlAgilityPack.XmlDocument returnDataXml = new HtmlAgilityPack.XmlDocument();
returnDataXml.LoadXml(returnData);
foreach (HtmlNode itemNode in returnDataXml.DocumentNode.SelectNodes("//item") ?? new HtmlNodeCollection(null)) {
string idStr = itemNode.GetAttributeValue("id", "");
if (!string.IsNullOrEmpty(idStr)) {
int id = int.Parse(idStr);
string conceptUri = MailData.SQLGetEntryId(id);
// add the concept uri
if (!string.IsNullOrEmpty(conceptUri))
itemNode.SetAttributeValue("uri", conceptUri.EncodeXMLString());
// add the label
if (_ao.ConceptUriToLabels.ContainsKey(conceptUri) && _ao.ConceptUriToLabels[conceptUri].Count > 0)
itemNode.SetAttributeValue("label", _ao.ConceptUriToLabels[conceptUri][0].EncodeXMLString());
}
}
returnData = returnDataXml.DocumentNode.InnerHtml;
returnData = returnData.Replace("<?xml version=\"1.0\" encoding=\"utf-8\"?>", "");
}
//else if (requestType == "GetItemDataForSimilarThreads")
//{
// var paramNode = requestNode.SelectSingleNode("./s1:requestData/query");
// int threadId = paramNode.GetAttributeValue("threadId", 0);
// int maxCount = paramNode.GetAttributeValue("maxCount", 20);
// int includeOnlyFirstInThread = paramNode.GetAttributeValue("includeOnlyFirstInThread", 1); // do we only want to return itemData for bug description or also for comments?
// int offset = paramNode.GetAttributeValue("offset", 0);
// returnData = GetItemDataForSimilarThreads(threadId, maxCount, offset, includeOnlyFirstInThread == 1);
//}
//else if (requestType == "GetItemDataForSimilarItems")
//{
// var paramNode = requestNode.SelectSingleNode("./s1:requestData/query");
// int threadId = paramNode.GetAttributeValue("itemId", 0);
// int maxCount = paramNode.GetAttributeValue("maxCount", 20);
// int offset = paramNode.GetAttributeValue("offset", 0);
// returnData = GetItemDataForSimilarItems(threadId, maxCount, offset);
//}
else if (requestType == "GetTagInfo") {
var paramNode = requestNode.SelectSingleNode("./s1:requestData/params");
int startIndex = paramNode.GetAttributeValue("startIndex", 0);
int count = paramNode.GetAttributeValue("count", 1000);
IEnumerable<TagInfoBase> tags = GetTagInfoList(startIndex, count);
StringBuilder text = new StringBuilder();
foreach (var tag in tags)
text.Append(tag.ToXML());
returnData = text.ToString();
}
else if (requestType == "ExecuteCommand")
returnData = ExecuteCommand(requestData);
else if (requestType == "SetData")
returnData = SetData(requestData);
else if (requestType == "RequestData")
returnData = RequestData(requestData);
else
AddEvent("Ignoring unknown requestType:" + requestType);
AddEvent("KEUI Indexing processed " + requestType + " event.");
return returnData;
}
void AQSubscriberWikiPostDeleted_OnMessageReceived(string message)
{
try {
lock (_mqLock) {
StoreEventData(_activeMQSettings.TopicNameWikiPostDeleted, message);
IncreaseProcessedCount();
HtmlAgilityPack.XmlDocument xmlDoc = new HtmlAgilityPack.XmlDocument();
xmlDoc.LoadXml(message);
_keui.DeleteWikiPost(xmlDoc);
StoreEventData(_activeMQSettings.TopicNameWikiPostDeleted, message, true, "invalid_");
}
}
catch (Exception ex) {
AddEvent("Exception while processing wiki deleted event: " + ex.Message);
GenLib.Log.LogService.LogException("Exception while processing wiki deleted event. message: " + message, ex);
}
}
public void OnPostPageDownloaded(object sender, PageCompletedEventArgs e)
{
if (!CanRun)
return;
try {
_processedCountSinceLastCheck++;
HtmlNode item = e.UserState as HtmlNode;
string link = e.SmallUri.GetString();
Dictionary<string, string> nameValueCollection = GenLib.Web.Utility.ParseQueryArguments(link);
int postId = -1;
int threadId = -1;
int forumId = -1;
if (nameValueCollection.ContainsKey("p"))
postId = int.Parse(nameValueCollection["p"]);
if (nameValueCollection.ContainsKey("t"))
threadId = int.Parse(nameValueCollection["t"]);
if (nameValueCollection.ContainsKey("f"))
forumId = int.Parse(nameValueCollection["f"]);
//if (!linkToPageContent.ContainsKey(link))
// linkToPageContent[link] = e.PageContent;
HtmlAgilityPack.XmlDocument doc = new HtmlAgilityPack.XmlDocument();
doc.LoadXml(e.PageContent);
var bodyNode = doc.DocumentNode.SelectSingleNode(String.Format("//div[@id='article{0}']", postId));
if (bodyNode == null) {
ReportError("Body for post page was null. Error downloading post information.");
return;
}
string body = bodyNode.InnerHtml;
body = body.StripHtml(); // todo: test if this removes everything ok
//body = body.Replace("\n", " ");
//body = Regex.Replace(body, "<br[ ]*[/]?>", Environment.NewLine);
//body = body.Replace(" \r\n", "\r\n");
//body = body.DecodeXMLString();
//body = RemoveBlockQuotes(body);
//body = System.Web.HttpUtility.HtmlDecode(body);
// use item and body to create an event that is sent
string subject = item.Element("title").InnerText;
string author = item.Element("author").InnerText;
string category = item.Element("category").InnerText;
string dateStr = item.Element("pubDate").InnerText;
string postIdStr = nameValueCollection["p"];
DateTime date = DateTime.Parse(dateStr);
//AddEvent("Parsed post with id " + postId);
//Trace.WriteLine("Parsed post " + postId);
AccountInfo account = AccountInfoList[_currentAccountIndex];
string eventData = Defaults.BuildNewForumItem(account.ForumId, account.ForumName, forumId, postId, threadId, link, date, subject, body, author, category);
if (DownloadTextOnly) {
string fileName = Path.Combine(DownloadPath, postId.ToString());
File.WriteAllText(fileName, eventData);
}
else {
lock (_publisherLock) {
StoreEventData(_activeMQSettings.TopicNameForumSensorPublishForumPost, eventData, false);
AQPublisher.SendMessage(eventData);
}
GenLib.Log.LogService.LogInfo("Parsed post with id " + postId);
}
lock (_postHashLock) {
_forumIdToImportedIdH[account.ForumId].Add(postIdStr);
}
if (postId % 10 == 0)
AddEvent("downloaded post with id " + postId);
}
catch (Exception ex) {
AddEvent("Exception while publishing new data: " + ex.Message);
GenLib.Log.LogService.LogException("ForumSensorDialog.OnPostPageDownloaded Exception while publishing new data: ", ex);
}
//string currentFolder = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location);
//File.WriteAllText(Path.Combine(currentFolder, itemIdStr + ".xml"), eventData);
_totalSentCount++;
//this.Invoke((Action)(() => { LabelStatus.Text = String.Format("Sent {0} posts", _totalSentCount); }));
}
//private Dictionary<string, string> linkToPageContent = new Dictionary<string, string>();
public void OnRSSPageDownloaded(object sender, PageCompletedEventArgs e)
{
try {
if (AQPublisher == null && !DownloadTextOnly) {
AddEvent("Active MQ publisher is null. Unable to post the event to the queue.");
return;
}
if (!CanRun)
return;
int currentForumId = AccountInfoList[_currentAccountIndex].ForumId;
XmlDocument doc = new HtmlAgilityPack.XmlDocument();
doc.LoadXml(e.PageContent);
foreach (HtmlNode item in doc.DocumentNode.SelectNodes("//channel/item") ?? new HtmlNodeCollection(null)) {
string link = item.Element("link").InnerText.DecodeXMLString();
if (link.LastIndexOf('#') >= 0)
link = link.Substring(0, link.LastIndexOf('#'));
Dictionary<string, string> nameValueCollection = Utility.ParseQueryArguments(link);
string itemId = "";
if (!nameValueCollection.ContainsKey("p")) {
AddEvent("A post in the RSS feed didn't contain post id. Ignoring the post. RSS feed: " + e.SmallUri.GetString());
continue;
}
itemId = nameValueCollection["p"];
lock (_postHashLock) {
if (_forumIdToImportedIdH[currentForumId].Contains(itemId))
continue;
}
WebGetter.DownloadPageAsync(this, item, new GenLib.Web.SmallUri(link), OnPostPageDownloaded);
}
//AddEvent("Parsed RSS page: " + e.SmallUri.GetString());
//Trace.WriteLine("Parsed RSS page: " + e.SmallUri.GetString());
GenLib.Log.LogService.LogInfo("Parsed RSS page " + e.SmallUri.GetString());
if (CanRun)
DownloadNextRSSPage();
}
catch (Exception ex) {
AddEvent("Exception while publishing new data: " + ex.Message);
GenLib.Log.LogService.LogException("ForumSensorDialog.OnPostPageDownloaded Exception while publishing new data: ", ex);
}
}
public void LoadSettings()
{
//string currentFolder = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location);
try {
// modify the default template folder
Defaults.TemplateFolder = Path.Combine(_forumSettingsFolder, "Event Templates");
if (!Directory.Exists(_forumSettingsFolder))
Directory.CreateDirectory(_forumSettingsFolder);
if (!Directory.Exists(Defaults.TemplateFolder))
ReportError(String.Format("The folder with templates ({0}) does not exist. Create it and put templates there.", Defaults.TemplateFolder));
_activeMQSettings = new ActiveMQSettings(Path.Combine(_forumSettingsFolder, _fileNameActiveMQSettings));
if (!string.IsNullOrEmpty(_activeMQSettings.LastInfo))
AddEvent("Error while loading Active MQ settings: " + _activeMQSettings.LastInfo);
else
AddEvent("Active MQ settings loaded successfully.");
_forumIdToImportedIdH = new Dictionary<int, HashSet<string>>();
AccountInfoList = new List<AccountInfo>();
if (File.Exists(Path.Combine(_forumSettingsFolder, _settingsFileName))) {
string text = File.ReadAllText(Path.Combine(_forumSettingsFolder, _settingsFileName));
HtmlAgilityPack.XmlDocument xmlDoc = new HtmlAgilityPack.XmlDocument();
xmlDoc.LoadXml(text);
var forumsNode = xmlDoc.DocumentNode.SelectSingleNode("/settings/forums");
CheckEveryMin = forumsNode.GetAttributeValue("checkEveryMin", 5);
SaveIdsEveryMin = forumsNode.GetAttributeValue("saveIdsEveryMin", 360);
StartOnStartup = forumsNode.GetAttributeValue("startOnStartup", true);
NrOfThreads = forumsNode.GetAttributeValue("nrOfThreads", 5);
foreach (var account in forumsNode.SelectNodes("./forum") ?? new HtmlNodeCollection(null)) {
int forumId = account.GetAttributeValue("forumId", -999);
string address = account.GetAttributeValue("address", "");
int checkBackDays = account.GetAttributeValue("checkBackDays", 1);
string forumName = account.GetAttributeValue("forumName", "");
if (forumId == -999) {
ReportError("No \"forumId\" value was specified for forum " + address + ". Since the attribute value is mandatory the forum is currently not being indexed.");
continue;
}
if (string.IsNullOrEmpty(address)) {
ReportError("The \"address\" value is not specified or contains an invalid value for forum with id " + forumId.ToString() + ". The forum is currently not being indexed.");
continue;
}
if (string.IsNullOrEmpty(forumName)) {
ReportError("The forumName value for forum " + address + " is empty. The value is mandatory so the forum is currently not being indexed.");
continue;
}
if (!(new[] { 0, 1, 7, 14, 30, 90, 180, 365 }).ToList().Contains(checkBackDays))
ReportError("The phpBB forum supports only a limited number of values for checkBackDays. These values are 0 (for whole history), 1, 7, 14, 30, 90, 180 and 365. Please specify in the settings one of these values otherwise the results could be unexpected.");
AccountInfoList.Add(new AccountInfo(forumId, forumName, address, checkBackDays));
// load the file with aleready imported forum post ids
lock (_postHashLock) {
string importedIdsFile = string.Format(_importedIdsFileTemplate, forumId);
if (File.Exists(Path.Combine(_forumSettingsFolder, importedIdsFile)))
_forumIdToImportedIdH[forumId] = new HashSet<string>(File.ReadAllLines(Path.Combine(_forumSettingsFolder, importedIdsFile)));
else
_forumIdToImportedIdH[forumId] = new HashSet<string>();
}
}
AddEvent(String.Format("The configuration file was read."));
AddEvent(String.Format("Data will be monitored on {0} sources.", AccountInfoList.Count));
AddEvent(String.Format("ActiveMQ Uri: {0}", _activeMQSettings.BrokerUri));
}
else
AddEvent("The configuration file was not found. No settings were loaded");
}
catch (Exception ex) {
GenLib.Log.LogService.LogException("Settings. LoadSettings failed.", ex);
AddEvent("Settings. LoadSettings failed." + ex.Message);
}
}
private string AddItemContent(QueryBase query, string returnData)
{
if (string.IsNullOrEmpty(returnData))
return returnData;
bool addItemData = false;
int itemDataSnipLen = -1;
bool snipMatchKeywords = true;
int keywordMatchOffset = 25;
List<string> keywords = new List<string>();
if (query is GeneralQuery) {
GeneralQuery generalQuery = query as GeneralQuery;
addItemData = generalQuery.QueryParams.ResultData.HasFlag(QResultData.itemData);
itemDataSnipLen = generalQuery.QueryParams.ItemDataSnipLen;
snipMatchKeywords = generalQuery.QueryParams.SnipMatchKeywords;
keywordMatchOffset = generalQuery.QueryParams.KeywordMatchOffset;
foreach (QKeywordCond kwCond in from arg in generalQuery.QueryArgs.Conditions where arg is QKeywordCond select arg)
keywords.AddRange(from kw in kwCond.Keywords select kw.Keyword);
}
else if (query is CustomQuery) {
CustomQuery customQuery = query as CustomQuery;
if (customQuery.QueryParams is QSimilarThreadsParam) {
addItemData = (customQuery.QueryParams as QSimilarThreadsParam).IncludeItemData;
itemDataSnipLen = (customQuery.QueryParams as QSimilarThreadsParam).ItemDataSnipLen;
}
else if (customQuery.QueryParams is QSimilarItemsParam) {
addItemData = (customQuery.QueryParams as QSimilarItemsParam).IncludeItemData;
itemDataSnipLen = (customQuery.QueryParams as QSimilarItemsParam).ItemDataSnipLen;
}
foreach (QKeywordCond kwCond in from arg in customQuery.QueryArgs.Conditions where arg is QKeywordCond select arg)
keywords.AddRange(from kw in kwCond.Keywords select kw.Keyword);
}
if (!addItemData)
return returnData;
HtmlAgilityPack.XmlDocument returnDoc = new HtmlAgilityPack.XmlDocument();
returnDoc.LoadXml(returnData);
foreach (HtmlAgilityPack.HtmlNode itemNode in returnDoc.DocumentNode.SelectNodes("//item") ?? new HtmlNodeCollection(null)) {
try {
int itemId = itemNode.GetAttributeValue("id", -1);
if (itemId == -1)
continue;
Tuple<string, string, string> subjectBodyMeta = SQLGetItemSubjectBodyMeta(itemId);
string subject = subjectBodyMeta.Item1;
string body = subjectBodyMeta.Item2;
string meta = subjectBodyMeta.Item3;
int firstIndex = -1;
string title;
if (itemDataSnipLen != -1) {
for (int i = 0; i < keywords.Count && i < 10; i++) {
int index = body.IndexOf(keywords[i], StringComparison.InvariantCultureIgnoreCase);
if (index >= 0 && (firstIndex == -1 || index < firstIndex))
firstIndex = index;
}
if (firstIndex > 0)
firstIndex -= keywordMatchOffset;
while (firstIndex > 0 && !char.IsWhiteSpace(body[firstIndex]))
firstIndex--;
if (firstIndex + itemDataSnipLen > body.Length)
firstIndex = body.Length - itemDataSnipLen;
firstIndex = Math.Max(0, firstIndex);
body = body.Substring(firstIndex, Math.Min(itemDataSnipLen, body.Length - firstIndex));
title = "shortContent";
}
else
title = "fullContent";
HtmlNode bodyNode = returnDoc.CreateElement(title);
bodyNode.AppendChild(returnDoc.CreateTextNode(body.EncodeXMLString()));
itemNode.AppendChild(bodyNode);
HtmlNode subjectNode = returnDoc.CreateElement("subject");
subjectNode.AppendChild(returnDoc.CreateTextNode(subject.EncodeXMLString()));
itemNode.AppendChild(subjectNode);
HtmlNode metaNode = returnDoc.CreateElement("metaData");
metaNode.AppendChild(returnDoc.CreateTextNode(meta));
itemNode.AppendChild(metaNode);
}
catch (Exception ex) {
GenLib.Log.LogService.LogException("MailData. AddItemContent: ", ex);
}
}
returnData = returnDoc.DocumentNode.InnerHtml;
return returnData;
}
public string GetItemInfo(int itemId = -1, string entryId = null)
{
DataRow data = null;
if (itemId != -1)
data = SQLGetItem(itemId);
else if (!string.IsNullOrEmpty(entryId))
data = SQLGetItem(entryId);
if (data == null)
return null;
if (itemId == -1)
itemId = (int)(long)data[0];
GeneralQuery generalQuery = new GeneralQuery(new QGeneralParams() { MaxCount = 1, IncludeAttachments = false, ItemDataSnipLen = -1 }, new QArgs(new QItemIdCond(itemId)));
string ret = MinerQuery(generalQuery);
if (string.IsNullOrEmpty(ret))
return null;
HtmlAgilityPack.XmlDocument doc = new HtmlAgilityPack.XmlDocument();
doc.LoadXml(ret);
HtmlAgilityPack.HtmlNode itemNode = doc.DocumentNode.SelectSingleNode("//item");
if (itemNode == null)
return null;
itemNode.SetAttributeValue("accountId", ((long)data[2]).ToString());
itemNode.SetAttributeValue("itemState", ((byte)data[4]).ToString());
return itemNode.OuterHtml;
}
void AQSubscriberKEUIRequest_OnMessageReceived(string message)
{
try
{
lock (_mqLock)
{
AddEvent("Received KEUI Request " + GetDateStamp());
StoreEventData(_activeMQSettings.TopicNameKEUIRequest, message);
HtmlAgilityPack.XmlDocument xmlDoc = new HtmlAgilityPack.XmlDocument();
xmlDoc.LoadXml(message);
var eventDataNode = xmlDoc.DocumentNode.SelectSingleNode("//ns1:eventData");
var requestNode = xmlDoc.DocumentNode.SelectSingleNode("//ns1:eventData/s1:keuiRequest");
if (requestNode == null) { AddEventAndLog("The KEUI request event did not contain //ns1:eventData/s1:keuiRequest node. Ignoring request"); return; }
string requestType = requestNode.SelectSingleNode("./s1:requestType").InnerText;
string publishTopic = requestNode.SelectSingleNode("./s1:publishResponseOnTopic") != null ? requestNode.SelectSingleNode("./s1:publishResponseOnTopic").InnerText.DecodeXMLString() : _activeMQSettings.TopicNameKEUIPublishResponse;
string keuiResponseData = null;
requestType = requestType.Trim(); // make sure we ignore the newlines and spaces
AddEvent("Request type: " + requestType);
if (requestType == "GetSuggestions")
{
var queryNode = requestNode.SelectSingleNode("./s1:requestData/query");
string prefix = queryNode.GetAttributeValue("prefix", "");
string suggestionTypes = queryNode.GetAttributeValue("suggestionTypes", "");
suggestionTypes = suggestionTypes.ToLower();
if (string.IsNullOrEmpty(prefix))
{
AddEventAndLog("Unable to provide suggestions. No prefix was specified in the event.");
return;
}
prefix = GenLib.Text.Text.ReplaceUnicodeCharsWithAscii(prefix);
keuiResponseData = "<suggestions>";
if (string.IsNullOrEmpty(suggestionTypes) || suggestionTypes.Contains("concepts"))
{
List<Tuple<string, string>> suggestionsConcepts = _keui.SuggestConceptsForPrefix(prefix);
foreach (var sugg in suggestionsConcepts)
keuiResponseData += String.Format("<concept label=\"{0}\" uri=\"{1}\" />", sugg.Item1.EncodeXMLString(), sugg.Item2.EncodeXMLString());
}
if (string.IsNullOrEmpty(suggestionTypes) || suggestionTypes.Contains("people"))
{
List<int> suggestionsPeople = _keui.SuggestPeopleForPrefix(prefix);
foreach (var personId in suggestionsPeople)
{
var personInfo = _keui.PeopleData.GetPerson(personId);
var accountIds = personInfo.GetAccountIds().ToList();
string account = "";
if (accountIds.Count > 0)
account = _keui.PeopleData.GetAccount(accountIds[0]);
string name = _keui.GetSuggestedPersonName(personInfo.PersonId);
keuiResponseData += String.Format("<person name=\"{0}\" account=\"{1}\" uuid=\"{2}\" />", name.EncodeXMLString(), account.EncodeXMLString(), personInfo.CustomData.EncodeXMLString());
}
}
if (string.IsNullOrEmpty(suggestionTypes) || suggestionTypes.Contains("products")) {
foreach (var sugg in _keui.SuggestProductsForPrefix(prefix))
keuiResponseData += String.Format("<product label=\"{0}\" uri=\"{1}\" />", sugg.Item1.EncodeXMLString(), sugg.Item2.EncodeXMLString());
}
if (string.IsNullOrEmpty(suggestionTypes) || suggestionTypes.Contains("issues")) {
foreach (var sugg in _keui.SuggestIssuesForPrefix(prefix))
keuiResponseData += String.Format("<issue label=\"{0}\" uri=\"{1}\" />", sugg.Item1.EncodeXMLString(), sugg.Item2.EncodeXMLString());
}
if (string.IsNullOrEmpty(suggestionTypes) || suggestionTypes.Contains("sources"))
{
foreach (int fileTag in _keui.SuggestFilesForPrefix(prefix)) {
string fileName = _keui.MailData.GetTagName(fileTag);
string fileUri = _keui.MailData.GetTagIdStr(fileTag);
fileUri = _keui.MakeUriLong(fileUri);
keuiResponseData += String.Format("<file name=\"{0}\" uri=\"{1}\" tooltip=\"{2}\" />", fileName.EncodeXMLString(), fileUri.EncodeXMLString(), fileName.EncodeXMLString());
}
foreach (int moduleTag in _keui.SuggestModulesForPrefix(prefix)) {
int fileTag = _keui.MailData.GetParentTagId(moduleTag);
string fileName = fileTag != TagInfoBase.InvalidTagId ? _keui.MailData.GetTagName(fileTag) : "";
string moduleUri = _keui.MailData.GetTagIdStr(moduleTag);
moduleUri = _keui.MakeUriLong(moduleUri);
keuiResponseData += String.Format("<module name=\"{0}\" uri=\"{1}\" tooltip=\"{2}\" />", _keui.MailData.GetTagName(moduleTag).EncodeXMLString(), moduleUri.EncodeXMLString(), fileName.EncodeXMLString());
}
foreach (int methodTag in _keui.SuggestMethodsForPrefix(prefix)) {
int moduleTag = _keui.MailData.GetParentTagId(methodTag);
int fileTag = moduleTag != TagInfoBase.InvalidTagId ? _keui.MailData.GetParentTagId(moduleTag) : TagInfoBase.InvalidTagId;
string fileName = fileTag != TagInfoBase.InvalidTagId ? _keui.MailData.GetTagName(fileTag) : "";
string methodUri = _keui.MailData.GetTagIdStr(methodTag);
methodUri = _keui.MakeUriLong(methodUri);
keuiResponseData += String.Format("<method name=\"{0}\" uri=\"{1}\" tooltip=\"{2}\" />", _keui.MailData.GetTagName(methodTag).EncodeXMLString(), methodUri.EncodeXMLString(), fileName.EncodeXMLString());
}
}
keuiResponseData += "</suggestions>";
}
else if (requestType == "GetAnnotationOntologyRDF")
{
var queryNode = requestNode.SelectSingleNode("./s1:requestData/query");
bool includeComment = queryNode != null && (queryNode.GetAttributeValue("includeComment", 0) == 1);
bool includeLinksTo = queryNode != null && (queryNode.GetAttributeValue("includeLinksTo", 0) == 1);
keuiResponseData = _keui.GetAnnotationOntologyRDF(includeComment, includeLinksTo);
AddEvent("Returning Annotation Ontology RDF. Data contains " + (int)(keuiResponseData.Length / 1024) + "KB");
}
else
keuiResponseData = _keui.ProcessKEUIRequest(xmlDoc, requestType);
if (keuiResponseData == null) return;
// update the event name
UpdateEventMetaData(xmlDoc, _activeMQSettings.TopicNameKEUIPublishResponse);
// insert the return data
_keui.AddNewLines(eventDataNode, 8);
eventDataNode.AppendChild(xmlDoc.CreateTextNode(String.Format("<s1:keuiResponse>{0}</s1:keuiResponse>", keuiResponseData)));
string returnEvent = xmlDoc.DocumentNode.OuterHtml;
// publish it
StoreEventData(publishTopic, returnEvent);
using (var publisher = new ActiveMqHelper.TopicPublisher(AQSession, publishTopic))
{
publisher.SendMessage(returnEvent);
}
AddEvent("KEUI successfully published KEUI Response event on topic " + publishTopic);
}
}
//catch (Apache.NMS.ActiveMQ.IOException ioe)
//{
// DisposeActiveMQ();
// InitActiveMQ();
//}
catch (Exception ex)
{
AddEvent("Exception while processing KEUI request: " + ex.Message);
GenLib.Log.LogService.LogException("Exception while processing KEUI request. message: " + message, ex);
}
}
public string RequestData(string query)
{
string ret = "";
DateTime start = DateTime.Now;
HtmlAgilityPack.XmlDocument queryDoc = new HtmlAgilityPack.XmlDocument();
queryDoc.LoadXml(query);
var queryNode = queryDoc.DocumentNode.SelectSingleNode("/query");
string source = queryNode.GetAttributeValue("source", null);
string content = queryNode.GetAttributeValue("content", null);
if (source == "PeopleData")
ret = PeopleData.RequestData(content, queryNode);
else if (source == "MailData")
ret = MailData.RequestData(content, queryNode);
else if (source == "TagData")
ret = MailData.RequestTagData(content, queryNode);
else
MailData.LogInfo("Warning: Don't know how to respond to query: " + query);
if (source != "MailData") {
MailData.LogInfo("RequestData query: " + query);
MailData.LogInfo(String.Format("RequestData needed {0:N0} ms, returning {1:N0} bytes", (DateTime.Now - start).TotalMilliseconds, ret.Length));
}
return ret;
}
void AQSubscriberTextToAnnotate_OnMessageReceived(string message)
{
try
{
lock (_mqLock)
{
StoreEventData(_activeMQSettings.TopicNameTextToAnnotate, message);
IncreaseProcessedCount();
HtmlAgilityPack.XmlDocument xmlDoc = new HtmlAgilityPack.XmlDocument();
xmlDoc.LoadXml(message);
_keui.AnnotateTextToAnnotate(xmlDoc);
UpdateEventMetaData(xmlDoc, _activeMQSettings.TopicNameKEUIPublishTextToAnnotate);
// publish it
string annotatedMessage = xmlDoc.DocumentNode.OuterHtml;
StoreEventData(_activeMQSettings.TopicNameKEUIPublishTextToAnnotate, annotatedMessage);
using (var AQPublisherTextToAnnotate = new ActiveMqHelper.TopicPublisher(AQSession, _activeMQSettings.TopicNameKEUIPublishTextToAnnotate))
{
AQPublisherTextToAnnotate.SendMessage(annotatedMessage);
}
}
}
catch (Exception ex)
{
AddEvent("Exception while annotating general text: " + ex.Message);
GenLib.Log.LogService.LogException("Exception while annotating general text. message: " + message, ex);
}
}
public string SetData(string data)
{
string ret = "";
DateTime start = DateTime.Now;
MailData.LogInfo("SetData data: " + data);
HtmlAgilityPack.XmlDocument dataDoc = new HtmlAgilityPack.XmlDocument();
dataDoc.LoadXml(data);
var dataNode = dataDoc.DocumentNode.SelectSingleNode("/data");
string target = dataNode.GetAttributeValue("target", null);
string content = dataNode.GetAttributeValue("content", null);
if (target == "MailData")
ret = MailData.SetData(content, dataNode);
else if (target == "TagData")
ret = MailData.SetTagData(content, dataNode);
else
MailData.LogInfo("Warning: Don't know how to use the data: " + data);
MailData.LogInfo(String.Format("SetData needed {0:N0} ms", (DateTime.Now - start).TotalMilliseconds));
return ret;
}
void AQSubscriberWikiPostNew_OnMessageReceived(string message)
{
try
{
lock (_mqLock)
{
StoreEventData(_activeMQSettings.TopicNameWikiPostNew, message);
IncreaseProcessedCount();
HtmlAgilityPack.XmlDocument xmlDoc = new HtmlAgilityPack.XmlDocument();
xmlDoc.LoadXml(message);
_keui.AnnotateWikiPost(xmlDoc);
int itemId = _keui.IndexWikiPost(xmlDoc);
if (itemId == -1)
StoreEventData(_activeMQSettings.TopicNameWikiPostNew, message, true, "invalid_");
UpdateEventMetaData(xmlDoc, _activeMQSettings.TopicNameKEUIPublishWikiPostNew);
// publish it
string annotatedMessage = xmlDoc.DocumentNode.OuterHtml;
StoreEventData(_activeMQSettings.TopicNameKEUIPublishWikiPostNew, annotatedMessage);
using (var publisher = new ActiveMqHelper.TopicPublisher(AQSession, _activeMQSettings.TopicNameKEUIPublishWikiPostNew))
{
publisher.SendMessage(annotatedMessage);
}
}
}
catch (Exception ex)
{
AddEvent("Exception while processing new wiki post: " + ex.Message);
GenLib.Log.LogService.LogException("Exception while processing new wiki post. message: " + message, ex);
}
}
public AddItemStatus AddItem(string itemInfo, string itemContent)
{
try {
itemInfo = Text.RemoveControlCharacters(itemInfo);
itemContent = Text.RemoveControlCharacters(itemContent);
if (string.IsNullOrEmpty(itemInfo))
return AddItemStatus.GetInvalidItemStatus("ItemInfo parameter is empty. Unable to add item to the index");
HtmlAgilityPack.XmlDocument itemInfoDoc = new HtmlAgilityPack.XmlDocument();
itemInfoDoc.LoadXml(itemInfo);
bool editedItemInfo = false;
HtmlNode itemNode = itemInfoDoc.DocumentNode.SelectSingleNode("/item");
if (itemNode.GetAttributeValue("itemId", -1) == -1) {
ESQLItemType itemType = (ESQLItemType)itemNode.GetAttributeValue("itemType", 0);
string entryId = itemInfoDoc.DocumentNode.SelectSingleNode("/item/entryId").InnerText.DecodeXMLString();
int accountId = itemNode.GetAttributeValue("accountId", -1);
SQLAddItem(entryId, itemType, accountId, ESQLItemState.ToBeAdded);
int computedItemId = SQLGetItemId(entryId);
itemNode.SetAttributeValue("itemId", computedItemId.ToString());
editedItemInfo = true;
}
if (itemInfoDoc.DocumentNode.SelectNodes("/item/people/person[@nameTrust]") != null) {
foreach (var node in itemInfoDoc.DocumentNode.SelectNodes("/item/people/person[@nameTrust]")) {
string account = node.GetAttributeValue("account", null);
string name = node.GetAttributeValue("name", null);
name = name.DecodeXMLString();
account = account.DecodeXMLString();
int accountType = node.GetAttributeValue("accountType", -1);
int nameTrust = node.GetAttributeValue("nameTrust", -1);
if (name == null || account == null || accountType == -1 || nameTrust == -1) {
GenLib.Log.LogService.LogError("Item was not added to the index due to missing information in the people section. Invalid data is: " + node.InnerHtml);
return AddItemStatus.GetInvalidItemStatus("Item was not added to the index due to missing information in the people section. Invalid data is: " + node.InnerHtml);
}
PeopleData.AddNewPersonOrUpdate(name, account, (EAccountType)accountType, (EPersonNameTrust)nameTrust);
int accountId = PeopleData.GetAccountId(account, (EAccountType)accountType);
node.SetAttributeValue("id", accountId.ToString());
editedItemInfo = true;
}
}
if (editedItemInfo == true)
itemInfo = itemInfoDoc.DocumentNode.InnerHtml;
int senderId = -1;
List<int> recipientIds = new List<int>();
foreach (var node in itemInfoDoc.DocumentNode.SelectNodes("/item/people/person") ?? new HtmlNodeCollection(null)) {
if (_senderRoles.Contains(node.GetAttributeValue("role", null))) senderId = node.GetAttributeValue("id", -1);
else if (_recipientRoles.Contains(node.GetAttributeValue("role", null))) recipientIds.Add(node.GetAttributeValue("id", -1));
}
if (senderId != -1) {
PeopleData.IncreaseCountInfo(senderId, recipientIds);
}
var subjectNode = itemNode.SelectSingleNode("./subject");
string subject = subjectNode != null ? subjectNode.InnerText.DecodeXMLString() : "";
var metaNode = itemNode.SelectSingleNode("./metaData");
string meta = metaNode != null ? metaNode.InnerHtml : "";
if (Settings.StoreToFileAddItemInfo)
StoreToFile("itemInfo", itemInfo);
if (Settings.StoreToFileAddItemContent)
StoreToFile("itemContent", itemContent);
AddItemStatus itemStatus = MinerAddItem(itemInfo, itemContent);
if (itemStatus.ItemId != -1)
{
SQLSetItemState(itemStatus.ItemId, ESQLItemState.Indexed);
SQLSetItemSubjectBodyMeta(itemStatus.ItemId, subject, itemContent, meta);
}
else
LogInfo("Indexing of the item failed. Error info: " + itemStatus.ErrorMessage);
return itemStatus;
}
catch (Exception ex) {
GenLib.Log.LogService.LogException("MailData. AddItem: ", ex);
return AddItemStatus.GetInvalidItemStatus("MailData AddItem exception: " + ex.Message, ex.StackTrace);
}
}
