void AQSubscriberIdentitySnapshot_OnMessageReceived(string message) { try { lock (_mqLock) { StoreEventData(_activeMQSettings.TopicNameIdentitySnapshot, message); IncreaseProcessedCount(); HtmlAgilityPack.XmlDocument xmlDoc = new HtmlAgilityPack.XmlDocument(); xmlDoc.LoadXml(message); _keui.IdentitySnapshot(xmlDoc); } } catch (Exception ex) { AddEvent("Exception while processing custom item to index: " + ex.Message); GenLib.Log.LogService.LogException("Exception while processing custom item to index. message: " + message, ex); } }
void AQSubscriberIssueUpdate_OnMessageReceived(string message) { try { lock (_mqLock) { StoreEventData(_activeMQSettings.TopicNameIssueUpdate, message); IncreaseProcessedCount(); HtmlAgilityPack.XmlDocument xmlDoc = new HtmlAgilityPack.XmlDocument(); xmlDoc.LoadXml(message); _keui.AnnotateIssueUpdate(xmlDoc); bool ok = _keui.IndexIssueUpdate(xmlDoc); if (!ok) StoreEventData(_activeMQSettings.TopicNameIssueUpdate, message, true, "invalid_"); UpdateEventMetaData(xmlDoc, _activeMQSettings.TopicNameKEUIPublishBugComment); // publish it string annotatedMessage = xmlDoc.DocumentNode.OuterHtml; StoreEventData(_activeMQSettings.TopicNameKEUIPublishBugComment, annotatedMessage); using (var publisher = new ActiveMqHelper.TopicPublisher(AQSession, _activeMQSettings.TopicNameKEUIPublishBugComment)) { publisher.SendMessage(annotatedMessage); } } } catch (Exception ex) { AddEvent("Exception while annotating new bug comment: " + ex.Message); GenLib.Log.LogService.LogException("Exception while processing new bug comment. message: " + message, ex); } }
public void LoadSettings() { //string currentFolder = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location); string currentFolder = SettingsServer.AppFolder; AddEvent("Loading settings from folder " + currentFolder); try { if (!Directory.Exists(Defaults.TemplateFolder)) AddEvent(String.Format("The folder with templates ({0}) does not exist. Create it and put templates there.", Defaults.TemplateFolder)); _activeMQSettings = new ActiveMQSettings(Path.Combine(currentFolder, _fileNameActiveMQSettings)); if (!string.IsNullOrEmpty(_activeMQSettings.LastInfo)) AddEvent("Error while loading Active MQ settings: " + _activeMQSettings.LastInfo); else AddEvent("Active MQ settings loaded successfully."); if (File.Exists(Path.Combine(currentFolder, _fileNameSettings))) { string text = File.ReadAllText(Path.Combine(currentFolder, _fileNameSettings)); HtmlAgilityPack.XmlDocument xmlDoc = new HtmlAgilityPack.XmlDocument(); xmlDoc.LoadXml(text); var indexingNode = xmlDoc.DocumentNode.SelectSingleNode("/settings/indexingService"); if (indexingNode != null) _defaultProfileName = indexingNode.GetAttributeValue("profileName", "ProfileData"); var pathNode = xmlDoc.DocumentNode.SelectSingleNode("/settings/paths"); if (pathNode != null) { _keui.IgnoredConceptsFileName = pathNode.GetAttributeValue("ignoredConcepts", _keui.IgnoredConceptsFileName); if (!_keui.IgnoredConceptsFileName.Contains(Path.DirectorySeparatorChar)) _keui.IgnoredConceptsFileName = Path.Combine(currentFolder, _keui.IgnoredConceptsFileName); _keui.CustomLemmasFileName = pathNode.GetAttributeValue("customLemmas", _keui.CustomLemmasFileName); if (!_keui.CustomLemmasFileName.Contains(Path.DirectorySeparatorChar)) _keui.CustomLemmasFileName = Path.Combine(currentFolder, _keui.CustomLemmasFileName); } } _keui.LoadSettings(currentFolder); } catch (Exception ex) { GenLib.Log.LogService.LogException("KEUI.LoadSettings failed.", ex); AddEvent("KEUI.SaveSettings failed." + ex.Message); } }
void AQSubscriberConceptNew_OnMessageReceived(string message) { try { lock (_mqLock) { AddEvent("Adding new rdf data to the annotation ontology. " + GetDateStamp()); StoreEventData(_activeMQSettings.TopicNameConceptNew, message); HtmlAgilityPack.XmlDocument xmlDoc = new HtmlAgilityPack.XmlDocument(); xmlDoc.LoadXml(message); var newRDFDataNode = xmlDoc.DocumentNode.SelectSingleNode("//ns1:eventData/ns1:newRDFData"); if (newRDFDataNode == null) { AddEventAndLog("The event did not contain //ns1:eventData/s1:newRDFData node. Ignoring event"); return; } string rdfData = newRDFDataNode.InnerHtml; rdfData = rdfData.Trim(); // make sure the <xml > tag is in the first line if (!rdfData.StartsWith("<?xml version=\"1.0\"?>")) rdfData = "<?xml version=\"1.0\"?>" + Environment.NewLine + rdfData; _keui.AddNewRDFData(rdfData); // todo: maybe also use the new concepts to tag existing data } } catch (Exception ex) { AddEvent("Exception while processing new RDF data: " + ex.Message); GenLib.Log.LogService.LogException("Exception while processing new RDF data. message: " + message, ex); } }
public string AddPeopleInfoToResults(string returnData) { try { if (string.IsNullOrEmpty(returnData)) return returnData; HtmlAgilityPack.XmlDocument returnDoc = new HtmlAgilityPack.XmlDocument(); returnDoc.LoadXml(returnData); char[] minusArray = "-".ToCharArray(); char[] commaArray = ",".ToCharArray(); HashSet<int> detectedAccountIds = new HashSet<int>(); string[] roles = ProfileSettings.PeopleRoles.Split(commaArray, StringSplitOptions.RemoveEmptyEntries); foreach (HtmlAgilityPack.HtmlNode itemNode in returnDoc.DocumentNode.SelectNodes("//item") ?? new HtmlNodeCollection(null)) { foreach (string role in roles) { string[] ids = itemNode.GetAttributeValue(role, "").Split(commaArray, StringSplitOptions.RemoveEmptyEntries); foreach (string id in ids) detectedAccountIds.Add(int.Parse(id)); } } var peopleNode = returnDoc.DocumentNode.SelectSingleNode("/results/fromToCounts"); if (peopleNode != null) { string[] items = peopleNode.InnerText.RemoveWhiteSpaces().Split(commaArray, StringSplitOptions.RemoveEmptyEntries); foreach (string item in items) { string[] parts = item.Split(minusArray, StringSplitOptions.RemoveEmptyEntries); detectedAccountIds.Add(int.Parse(parts[0])); detectedAccountIds.Add(int.Parse(parts[1])); } } if (detectedAccountIds.Count == 0) return returnData; StringBuilder peopleInfo = new StringBuilder(); peopleInfo.Append("\t<peopleData>\n"); foreach (int id in detectedAccountIds) { IPersonInfo p = PeopleData.GetPersonFromAccount(id); peopleInfo.Append(String.Format("\t\t<person id=\"{0}\" account=\"{1}\" name=\"{2}\" uuid=\"{3}\" />\n", id, PeopleData.GetAccount(id).EncodeXMLString(), p.Name.EncodeXMLString(), p.CustomData.EncodeXMLString())); } peopleInfo.Append("\t</peopleData>\n"); Match resultsMatch = Regex.Match(returnData, @"<results[^>]*>\n"); Trace.Assert(resultsMatch.Success == true, "Unable to find <results> section in the result data"); int insertIndex = resultsMatch.Index + resultsMatch.Length; return returnData.Insert(insertIndex, peopleInfo.ToString()); } catch (Exception ex) { GenLib.Log.LogService.LogException("MailData. AddPeopleInfoToResults: ", ex); } return ""; }
public string ExecuteCommand(string commandData) { string ret = ""; if (string.IsNullOrEmpty(commandData)) return ret; DateTime start = DateTime.Now; MailData.LogInfo("ExecuteCommand command: " + commandData.Trim()); HtmlAgilityPack.XmlDocument commandDoc = new HtmlAgilityPack.XmlDocument(); commandDoc.LoadXml(commandData); var commandNode = commandDoc.DocumentNode.SelectSingleNode("/commandData"); string target = commandNode.GetAttributeValue("target", null); string command = commandNode.GetAttributeValue("command", null); if (target == "PeopleData") ret = PeopleData.ExecuteCommand(command, commandNode); else if (target == "MailData") ret = MailData.ExecuteCommand(command, commandNode); else if (target == "Miner") { ret = MailData.MinerExecuteCommand(commandData); GenLib.Log.LogService.LogInfo("Profile. ExecuteCommand result: " + ret); } else MailData.LogInfo("Warning: Don't know how to process the command: " + commandData); MailData.LogInfo(String.Format("ExecuteCommand needed {0:N0} ms, returning {1:N0} bytes", (DateTime.Now - start).TotalMilliseconds, ret.Length)); return ret; }
public string ProcessKEUIRequest(HtmlAgilityPack.XmlDocument xmlDoc, string requestType) { Debug.Assert(MailData != null, "MailData is null. The indexing service has to be initialize before requests can be processed."); var requestNode = xmlDoc.DocumentNode.SelectSingleNode("//ns1:eventData/s1:keuiRequest"); string requestData = requestNode.SelectSingleNode("./s1:requestData").InnerHtml; //requestData = requestData.Replace("<s1:", "<").Replace("</s1:", "</"); var sequenceNumberNode = xmlDoc.DocumentNode.SelectSingleNode("//ns1:head/ns1:sequencenumber"); if (sequenceNumberNode != null) Trace.WriteLine("Received request with sequencenumber: " + sequenceNumberNode.InnerText); string returnData = ""; if (requestType == "Query") { requestData = ChangeBugIdToThreadId(requestData); // if we have specified a bug id this will fix it to threadId requestData = FixQueryArgs(requestData); returnData = ProcessQuery(requestData); returnData = returnData.Replace("<?xml version=\"1.0\" encoding=\"utf-8\"?>", ""); } else if (requestType == "GetSimilarConcepts") { requestData = FixQueryArgs(requestData); returnData = ProcessQuery(requestData); HtmlAgilityPack.XmlDocument returnDataXml = new HtmlAgilityPack.XmlDocument(); returnDataXml.LoadXml(returnData); foreach (HtmlNode itemNode in returnDataXml.DocumentNode.SelectNodes("//item") ?? new HtmlNodeCollection(null)) { string idStr = itemNode.GetAttributeValue("id", ""); if (!string.IsNullOrEmpty(idStr)) { int id = int.Parse(idStr); string conceptUri = MailData.SQLGetEntryId(id); // add the concept uri if (!string.IsNullOrEmpty(conceptUri)) itemNode.SetAttributeValue("uri", conceptUri.EncodeXMLString()); // add the label if (_ao.ConceptUriToLabels.ContainsKey(conceptUri) && _ao.ConceptUriToLabels[conceptUri].Count > 0) itemNode.SetAttributeValue("label", _ao.ConceptUriToLabels[conceptUri][0].EncodeXMLString()); } } returnData = returnDataXml.DocumentNode.InnerHtml; returnData = returnData.Replace("<?xml version=\"1.0\" encoding=\"utf-8\"?>", ""); } //else if (requestType == "GetItemDataForSimilarThreads") //{ // var paramNode = requestNode.SelectSingleNode("./s1:requestData/query"); // int threadId = paramNode.GetAttributeValue("threadId", 0); // int maxCount = paramNode.GetAttributeValue("maxCount", 20); // int includeOnlyFirstInThread = paramNode.GetAttributeValue("includeOnlyFirstInThread", 1); // do we only want to return itemData for bug description or also for comments? // int offset = paramNode.GetAttributeValue("offset", 0); // returnData = GetItemDataForSimilarThreads(threadId, maxCount, offset, includeOnlyFirstInThread == 1); //} //else if (requestType == "GetItemDataForSimilarItems") //{ // var paramNode = requestNode.SelectSingleNode("./s1:requestData/query"); // int threadId = paramNode.GetAttributeValue("itemId", 0); // int maxCount = paramNode.GetAttributeValue("maxCount", 20); // int offset = paramNode.GetAttributeValue("offset", 0); // returnData = GetItemDataForSimilarItems(threadId, maxCount, offset); //} else if (requestType == "GetTagInfo") { var paramNode = requestNode.SelectSingleNode("./s1:requestData/params"); int startIndex = paramNode.GetAttributeValue("startIndex", 0); int count = paramNode.GetAttributeValue("count", 1000); IEnumerable<TagInfoBase> tags = GetTagInfoList(startIndex, count); StringBuilder text = new StringBuilder(); foreach (var tag in tags) text.Append(tag.ToXML()); returnData = text.ToString(); } else if (requestType == "ExecuteCommand") returnData = ExecuteCommand(requestData); else if (requestType == "SetData") returnData = SetData(requestData); else if (requestType == "RequestData") returnData = RequestData(requestData); else AddEvent("Ignoring unknown requestType:" + requestType); AddEvent("KEUI Indexing processed " + requestType + " event."); return returnData; }
void AQSubscriberWikiPostDeleted_OnMessageReceived(string message) { try { lock (_mqLock) { StoreEventData(_activeMQSettings.TopicNameWikiPostDeleted, message); IncreaseProcessedCount(); HtmlAgilityPack.XmlDocument xmlDoc = new HtmlAgilityPack.XmlDocument(); xmlDoc.LoadXml(message); _keui.DeleteWikiPost(xmlDoc); StoreEventData(_activeMQSettings.TopicNameWikiPostDeleted, message, true, "invalid_"); } } catch (Exception ex) { AddEvent("Exception while processing wiki deleted event: " + ex.Message); GenLib.Log.LogService.LogException("Exception while processing wiki deleted event. message: " + message, ex); } }
public void OnPostPageDownloaded(object sender, PageCompletedEventArgs e) { if (!CanRun) return; try { _processedCountSinceLastCheck++; HtmlNode item = e.UserState as HtmlNode; string link = e.SmallUri.GetString(); Dictionary<string, string> nameValueCollection = GenLib.Web.Utility.ParseQueryArguments(link); int postId = -1; int threadId = -1; int forumId = -1; if (nameValueCollection.ContainsKey("p")) postId = int.Parse(nameValueCollection["p"]); if (nameValueCollection.ContainsKey("t")) threadId = int.Parse(nameValueCollection["t"]); if (nameValueCollection.ContainsKey("f")) forumId = int.Parse(nameValueCollection["f"]); //if (!linkToPageContent.ContainsKey(link)) // linkToPageContent[link] = e.PageContent; HtmlAgilityPack.XmlDocument doc = new HtmlAgilityPack.XmlDocument(); doc.LoadXml(e.PageContent); var bodyNode = doc.DocumentNode.SelectSingleNode(String.Format("//div[@id='article{0}']", postId)); if (bodyNode == null) { ReportError("Body for post page was null. Error downloading post information."); return; } string body = bodyNode.InnerHtml; body = body.StripHtml(); // todo: test if this removes everything ok //body = body.Replace("\n", " "); //body = Regex.Replace(body, "<br[ ]*[/]?>", Environment.NewLine); //body = body.Replace(" \r\n", "\r\n"); //body = body.DecodeXMLString(); //body = RemoveBlockQuotes(body); //body = System.Web.HttpUtility.HtmlDecode(body); // use item and body to create an event that is sent string subject = item.Element("title").InnerText; string author = item.Element("author").InnerText; string category = item.Element("category").InnerText; string dateStr = item.Element("pubDate").InnerText; string postIdStr = nameValueCollection["p"]; DateTime date = DateTime.Parse(dateStr); //AddEvent("Parsed post with id " + postId); //Trace.WriteLine("Parsed post " + postId); AccountInfo account = AccountInfoList[_currentAccountIndex]; string eventData = Defaults.BuildNewForumItem(account.ForumId, account.ForumName, forumId, postId, threadId, link, date, subject, body, author, category); if (DownloadTextOnly) { string fileName = Path.Combine(DownloadPath, postId.ToString()); File.WriteAllText(fileName, eventData); } else { lock (_publisherLock) { StoreEventData(_activeMQSettings.TopicNameForumSensorPublishForumPost, eventData, false); AQPublisher.SendMessage(eventData); } GenLib.Log.LogService.LogInfo("Parsed post with id " + postId); } lock (_postHashLock) { _forumIdToImportedIdH[account.ForumId].Add(postIdStr); } if (postId % 10 == 0) AddEvent("downloaded post with id " + postId); } catch (Exception ex) { AddEvent("Exception while publishing new data: " + ex.Message); GenLib.Log.LogService.LogException("ForumSensorDialog.OnPostPageDownloaded Exception while publishing new data: ", ex); } //string currentFolder = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location); //File.WriteAllText(Path.Combine(currentFolder, itemIdStr + ".xml"), eventData); _totalSentCount++; //this.Invoke((Action)(() => { LabelStatus.Text = String.Format("Sent {0} posts", _totalSentCount); })); }
//private Dictionary<string, string> linkToPageContent = new Dictionary<string, string>(); public void OnRSSPageDownloaded(object sender, PageCompletedEventArgs e) { try { if (AQPublisher == null && !DownloadTextOnly) { AddEvent("Active MQ publisher is null. Unable to post the event to the queue."); return; } if (!CanRun) return; int currentForumId = AccountInfoList[_currentAccountIndex].ForumId; XmlDocument doc = new HtmlAgilityPack.XmlDocument(); doc.LoadXml(e.PageContent); foreach (HtmlNode item in doc.DocumentNode.SelectNodes("//channel/item") ?? new HtmlNodeCollection(null)) { string link = item.Element("link").InnerText.DecodeXMLString(); if (link.LastIndexOf('#') >= 0) link = link.Substring(0, link.LastIndexOf('#')); Dictionary<string, string> nameValueCollection = Utility.ParseQueryArguments(link); string itemId = ""; if (!nameValueCollection.ContainsKey("p")) { AddEvent("A post in the RSS feed didn't contain post id. Ignoring the post. RSS feed: " + e.SmallUri.GetString()); continue; } itemId = nameValueCollection["p"]; lock (_postHashLock) { if (_forumIdToImportedIdH[currentForumId].Contains(itemId)) continue; } WebGetter.DownloadPageAsync(this, item, new GenLib.Web.SmallUri(link), OnPostPageDownloaded); } //AddEvent("Parsed RSS page: " + e.SmallUri.GetString()); //Trace.WriteLine("Parsed RSS page: " + e.SmallUri.GetString()); GenLib.Log.LogService.LogInfo("Parsed RSS page " + e.SmallUri.GetString()); if (CanRun) DownloadNextRSSPage(); } catch (Exception ex) { AddEvent("Exception while publishing new data: " + ex.Message); GenLib.Log.LogService.LogException("ForumSensorDialog.OnPostPageDownloaded Exception while publishing new data: ", ex); } }
public void LoadSettings() { //string currentFolder = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location); try { // modify the default template folder Defaults.TemplateFolder = Path.Combine(_forumSettingsFolder, "Event Templates"); if (!Directory.Exists(_forumSettingsFolder)) Directory.CreateDirectory(_forumSettingsFolder); if (!Directory.Exists(Defaults.TemplateFolder)) ReportError(String.Format("The folder with templates ({0}) does not exist. Create it and put templates there.", Defaults.TemplateFolder)); _activeMQSettings = new ActiveMQSettings(Path.Combine(_forumSettingsFolder, _fileNameActiveMQSettings)); if (!string.IsNullOrEmpty(_activeMQSettings.LastInfo)) AddEvent("Error while loading Active MQ settings: " + _activeMQSettings.LastInfo); else AddEvent("Active MQ settings loaded successfully."); _forumIdToImportedIdH = new Dictionary<int, HashSet<string>>(); AccountInfoList = new List<AccountInfo>(); if (File.Exists(Path.Combine(_forumSettingsFolder, _settingsFileName))) { string text = File.ReadAllText(Path.Combine(_forumSettingsFolder, _settingsFileName)); HtmlAgilityPack.XmlDocument xmlDoc = new HtmlAgilityPack.XmlDocument(); xmlDoc.LoadXml(text); var forumsNode = xmlDoc.DocumentNode.SelectSingleNode("/settings/forums"); CheckEveryMin = forumsNode.GetAttributeValue("checkEveryMin", 5); SaveIdsEveryMin = forumsNode.GetAttributeValue("saveIdsEveryMin", 360); StartOnStartup = forumsNode.GetAttributeValue("startOnStartup", true); NrOfThreads = forumsNode.GetAttributeValue("nrOfThreads", 5); foreach (var account in forumsNode.SelectNodes("./forum") ?? new HtmlNodeCollection(null)) { int forumId = account.GetAttributeValue("forumId", -999); string address = account.GetAttributeValue("address", ""); int checkBackDays = account.GetAttributeValue("checkBackDays", 1); string forumName = account.GetAttributeValue("forumName", ""); if (forumId == -999) { ReportError("No \"forumId\" value was specified for forum " + address + ". Since the attribute value is mandatory the forum is currently not being indexed."); continue; } if (string.IsNullOrEmpty(address)) { ReportError("The \"address\" value is not specified or contains an invalid value for forum with id " + forumId.ToString() + ". The forum is currently not being indexed."); continue; } if (string.IsNullOrEmpty(forumName)) { ReportError("The forumName value for forum " + address + " is empty. The value is mandatory so the forum is currently not being indexed."); continue; } if (!(new[] { 0, 1, 7, 14, 30, 90, 180, 365 }).ToList().Contains(checkBackDays)) ReportError("The phpBB forum supports only a limited number of values for checkBackDays. These values are 0 (for whole history), 1, 7, 14, 30, 90, 180 and 365. Please specify in the settings one of these values otherwise the results could be unexpected."); AccountInfoList.Add(new AccountInfo(forumId, forumName, address, checkBackDays)); // load the file with aleready imported forum post ids lock (_postHashLock) { string importedIdsFile = string.Format(_importedIdsFileTemplate, forumId); if (File.Exists(Path.Combine(_forumSettingsFolder, importedIdsFile))) _forumIdToImportedIdH[forumId] = new HashSet<string>(File.ReadAllLines(Path.Combine(_forumSettingsFolder, importedIdsFile))); else _forumIdToImportedIdH[forumId] = new HashSet<string>(); } } AddEvent(String.Format("The configuration file was read.")); AddEvent(String.Format("Data will be monitored on {0} sources.", AccountInfoList.Count)); AddEvent(String.Format("ActiveMQ Uri: {0}", _activeMQSettings.BrokerUri)); } else AddEvent("The configuration file was not found. No settings were loaded"); } catch (Exception ex) { GenLib.Log.LogService.LogException("Settings. LoadSettings failed.", ex); AddEvent("Settings. LoadSettings failed." + ex.Message); } }
private string AddItemContent(QueryBase query, string returnData) { if (string.IsNullOrEmpty(returnData)) return returnData; bool addItemData = false; int itemDataSnipLen = -1; bool snipMatchKeywords = true; int keywordMatchOffset = 25; List<string> keywords = new List<string>(); if (query is GeneralQuery) { GeneralQuery generalQuery = query as GeneralQuery; addItemData = generalQuery.QueryParams.ResultData.HasFlag(QResultData.itemData); itemDataSnipLen = generalQuery.QueryParams.ItemDataSnipLen; snipMatchKeywords = generalQuery.QueryParams.SnipMatchKeywords; keywordMatchOffset = generalQuery.QueryParams.KeywordMatchOffset; foreach (QKeywordCond kwCond in from arg in generalQuery.QueryArgs.Conditions where arg is QKeywordCond select arg) keywords.AddRange(from kw in kwCond.Keywords select kw.Keyword); } else if (query is CustomQuery) { CustomQuery customQuery = query as CustomQuery; if (customQuery.QueryParams is QSimilarThreadsParam) { addItemData = (customQuery.QueryParams as QSimilarThreadsParam).IncludeItemData; itemDataSnipLen = (customQuery.QueryParams as QSimilarThreadsParam).ItemDataSnipLen; } else if (customQuery.QueryParams is QSimilarItemsParam) { addItemData = (customQuery.QueryParams as QSimilarItemsParam).IncludeItemData; itemDataSnipLen = (customQuery.QueryParams as QSimilarItemsParam).ItemDataSnipLen; } foreach (QKeywordCond kwCond in from arg in customQuery.QueryArgs.Conditions where arg is QKeywordCond select arg) keywords.AddRange(from kw in kwCond.Keywords select kw.Keyword); } if (!addItemData) return returnData; HtmlAgilityPack.XmlDocument returnDoc = new HtmlAgilityPack.XmlDocument(); returnDoc.LoadXml(returnData); foreach (HtmlAgilityPack.HtmlNode itemNode in returnDoc.DocumentNode.SelectNodes("//item") ?? new HtmlNodeCollection(null)) { try { int itemId = itemNode.GetAttributeValue("id", -1); if (itemId == -1) continue; Tuple<string, string, string> subjectBodyMeta = SQLGetItemSubjectBodyMeta(itemId); string subject = subjectBodyMeta.Item1; string body = subjectBodyMeta.Item2; string meta = subjectBodyMeta.Item3; int firstIndex = -1; string title; if (itemDataSnipLen != -1) { for (int i = 0; i < keywords.Count && i < 10; i++) { int index = body.IndexOf(keywords[i], StringComparison.InvariantCultureIgnoreCase); if (index >= 0 && (firstIndex == -1 || index < firstIndex)) firstIndex = index; } if (firstIndex > 0) firstIndex -= keywordMatchOffset; while (firstIndex > 0 && !char.IsWhiteSpace(body[firstIndex])) firstIndex--; if (firstIndex + itemDataSnipLen > body.Length) firstIndex = body.Length - itemDataSnipLen; firstIndex = Math.Max(0, firstIndex); body = body.Substring(firstIndex, Math.Min(itemDataSnipLen, body.Length - firstIndex)); title = "shortContent"; } else title = "fullContent"; HtmlNode bodyNode = returnDoc.CreateElement(title); bodyNode.AppendChild(returnDoc.CreateTextNode(body.EncodeXMLString())); itemNode.AppendChild(bodyNode); HtmlNode subjectNode = returnDoc.CreateElement("subject"); subjectNode.AppendChild(returnDoc.CreateTextNode(subject.EncodeXMLString())); itemNode.AppendChild(subjectNode); HtmlNode metaNode = returnDoc.CreateElement("metaData"); metaNode.AppendChild(returnDoc.CreateTextNode(meta)); itemNode.AppendChild(metaNode); } catch (Exception ex) { GenLib.Log.LogService.LogException("MailData. AddItemContent: ", ex); } } returnData = returnDoc.DocumentNode.InnerHtml; return returnData; }
public string GetItemInfo(int itemId = -1, string entryId = null) { DataRow data = null; if (itemId != -1) data = SQLGetItem(itemId); else if (!string.IsNullOrEmpty(entryId)) data = SQLGetItem(entryId); if (data == null) return null; if (itemId == -1) itemId = (int)(long)data[0]; GeneralQuery generalQuery = new GeneralQuery(new QGeneralParams() { MaxCount = 1, IncludeAttachments = false, ItemDataSnipLen = -1 }, new QArgs(new QItemIdCond(itemId))); string ret = MinerQuery(generalQuery); if (string.IsNullOrEmpty(ret)) return null; HtmlAgilityPack.XmlDocument doc = new HtmlAgilityPack.XmlDocument(); doc.LoadXml(ret); HtmlAgilityPack.HtmlNode itemNode = doc.DocumentNode.SelectSingleNode("//item"); if (itemNode == null) return null; itemNode.SetAttributeValue("accountId", ((long)data[2]).ToString()); itemNode.SetAttributeValue("itemState", ((byte)data[4]).ToString()); return itemNode.OuterHtml; }
void AQSubscriberKEUIRequest_OnMessageReceived(string message) { try { lock (_mqLock) { AddEvent("Received KEUI Request " + GetDateStamp()); StoreEventData(_activeMQSettings.TopicNameKEUIRequest, message); HtmlAgilityPack.XmlDocument xmlDoc = new HtmlAgilityPack.XmlDocument(); xmlDoc.LoadXml(message); var eventDataNode = xmlDoc.DocumentNode.SelectSingleNode("//ns1:eventData"); var requestNode = xmlDoc.DocumentNode.SelectSingleNode("//ns1:eventData/s1:keuiRequest"); if (requestNode == null) { AddEventAndLog("The KEUI request event did not contain //ns1:eventData/s1:keuiRequest node. Ignoring request"); return; } string requestType = requestNode.SelectSingleNode("./s1:requestType").InnerText; string publishTopic = requestNode.SelectSingleNode("./s1:publishResponseOnTopic") != null ? requestNode.SelectSingleNode("./s1:publishResponseOnTopic").InnerText.DecodeXMLString() : _activeMQSettings.TopicNameKEUIPublishResponse; string keuiResponseData = null; requestType = requestType.Trim(); // make sure we ignore the newlines and spaces AddEvent("Request type: " + requestType); if (requestType == "GetSuggestions") { var queryNode = requestNode.SelectSingleNode("./s1:requestData/query"); string prefix = queryNode.GetAttributeValue("prefix", ""); string suggestionTypes = queryNode.GetAttributeValue("suggestionTypes", ""); suggestionTypes = suggestionTypes.ToLower(); if (string.IsNullOrEmpty(prefix)) { AddEventAndLog("Unable to provide suggestions. No prefix was specified in the event."); return; } prefix = GenLib.Text.Text.ReplaceUnicodeCharsWithAscii(prefix); keuiResponseData = "<suggestions>"; if (string.IsNullOrEmpty(suggestionTypes) || suggestionTypes.Contains("concepts")) { List<Tuple<string, string>> suggestionsConcepts = _keui.SuggestConceptsForPrefix(prefix); foreach (var sugg in suggestionsConcepts) keuiResponseData += String.Format("<concept label=\"{0}\" uri=\"{1}\" />", sugg.Item1.EncodeXMLString(), sugg.Item2.EncodeXMLString()); } if (string.IsNullOrEmpty(suggestionTypes) || suggestionTypes.Contains("people")) { List<int> suggestionsPeople = _keui.SuggestPeopleForPrefix(prefix); foreach (var personId in suggestionsPeople) { var personInfo = _keui.PeopleData.GetPerson(personId); var accountIds = personInfo.GetAccountIds().ToList(); string account = ""; if (accountIds.Count > 0) account = _keui.PeopleData.GetAccount(accountIds[0]); string name = _keui.GetSuggestedPersonName(personInfo.PersonId); keuiResponseData += String.Format("<person name=\"{0}\" account=\"{1}\" uuid=\"{2}\" />", name.EncodeXMLString(), account.EncodeXMLString(), personInfo.CustomData.EncodeXMLString()); } } if (string.IsNullOrEmpty(suggestionTypes) || suggestionTypes.Contains("products")) { foreach (var sugg in _keui.SuggestProductsForPrefix(prefix)) keuiResponseData += String.Format("<product label=\"{0}\" uri=\"{1}\" />", sugg.Item1.EncodeXMLString(), sugg.Item2.EncodeXMLString()); } if (string.IsNullOrEmpty(suggestionTypes) || suggestionTypes.Contains("issues")) { foreach (var sugg in _keui.SuggestIssuesForPrefix(prefix)) keuiResponseData += String.Format("<issue label=\"{0}\" uri=\"{1}\" />", sugg.Item1.EncodeXMLString(), sugg.Item2.EncodeXMLString()); } if (string.IsNullOrEmpty(suggestionTypes) || suggestionTypes.Contains("sources")) { foreach (int fileTag in _keui.SuggestFilesForPrefix(prefix)) { string fileName = _keui.MailData.GetTagName(fileTag); string fileUri = _keui.MailData.GetTagIdStr(fileTag); fileUri = _keui.MakeUriLong(fileUri); keuiResponseData += String.Format("<file name=\"{0}\" uri=\"{1}\" tooltip=\"{2}\" />", fileName.EncodeXMLString(), fileUri.EncodeXMLString(), fileName.EncodeXMLString()); } foreach (int moduleTag in _keui.SuggestModulesForPrefix(prefix)) { int fileTag = _keui.MailData.GetParentTagId(moduleTag); string fileName = fileTag != TagInfoBase.InvalidTagId ? _keui.MailData.GetTagName(fileTag) : ""; string moduleUri = _keui.MailData.GetTagIdStr(moduleTag); moduleUri = _keui.MakeUriLong(moduleUri); keuiResponseData += String.Format("<module name=\"{0}\" uri=\"{1}\" tooltip=\"{2}\" />", _keui.MailData.GetTagName(moduleTag).EncodeXMLString(), moduleUri.EncodeXMLString(), fileName.EncodeXMLString()); } foreach (int methodTag in _keui.SuggestMethodsForPrefix(prefix)) { int moduleTag = _keui.MailData.GetParentTagId(methodTag); int fileTag = moduleTag != TagInfoBase.InvalidTagId ? _keui.MailData.GetParentTagId(moduleTag) : TagInfoBase.InvalidTagId; string fileName = fileTag != TagInfoBase.InvalidTagId ? _keui.MailData.GetTagName(fileTag) : ""; string methodUri = _keui.MailData.GetTagIdStr(methodTag); methodUri = _keui.MakeUriLong(methodUri); keuiResponseData += String.Format("<method name=\"{0}\" uri=\"{1}\" tooltip=\"{2}\" />", _keui.MailData.GetTagName(methodTag).EncodeXMLString(), methodUri.EncodeXMLString(), fileName.EncodeXMLString()); } } keuiResponseData += "</suggestions>"; } else if (requestType == "GetAnnotationOntologyRDF") { var queryNode = requestNode.SelectSingleNode("./s1:requestData/query"); bool includeComment = queryNode != null && (queryNode.GetAttributeValue("includeComment", 0) == 1); bool includeLinksTo = queryNode != null && (queryNode.GetAttributeValue("includeLinksTo", 0) == 1); keuiResponseData = _keui.GetAnnotationOntologyRDF(includeComment, includeLinksTo); AddEvent("Returning Annotation Ontology RDF. Data contains " + (int)(keuiResponseData.Length / 1024) + "KB"); } else keuiResponseData = _keui.ProcessKEUIRequest(xmlDoc, requestType); if (keuiResponseData == null) return; // update the event name UpdateEventMetaData(xmlDoc, _activeMQSettings.TopicNameKEUIPublishResponse); // insert the return data _keui.AddNewLines(eventDataNode, 8); eventDataNode.AppendChild(xmlDoc.CreateTextNode(String.Format("<s1:keuiResponse>{0}</s1:keuiResponse>", keuiResponseData))); string returnEvent = xmlDoc.DocumentNode.OuterHtml; // publish it StoreEventData(publishTopic, returnEvent); using (var publisher = new ActiveMqHelper.TopicPublisher(AQSession, publishTopic)) { publisher.SendMessage(returnEvent); } AddEvent("KEUI successfully published KEUI Response event on topic " + publishTopic); } } //catch (Apache.NMS.ActiveMQ.IOException ioe) //{ // DisposeActiveMQ(); // InitActiveMQ(); //} catch (Exception ex) { AddEvent("Exception while processing KEUI request: " + ex.Message); GenLib.Log.LogService.LogException("Exception while processing KEUI request. message: " + message, ex); } }
public string RequestData(string query) { string ret = ""; DateTime start = DateTime.Now; HtmlAgilityPack.XmlDocument queryDoc = new HtmlAgilityPack.XmlDocument(); queryDoc.LoadXml(query); var queryNode = queryDoc.DocumentNode.SelectSingleNode("/query"); string source = queryNode.GetAttributeValue("source", null); string content = queryNode.GetAttributeValue("content", null); if (source == "PeopleData") ret = PeopleData.RequestData(content, queryNode); else if (source == "MailData") ret = MailData.RequestData(content, queryNode); else if (source == "TagData") ret = MailData.RequestTagData(content, queryNode); else MailData.LogInfo("Warning: Don't know how to respond to query: " + query); if (source != "MailData") { MailData.LogInfo("RequestData query: " + query); MailData.LogInfo(String.Format("RequestData needed {0:N0} ms, returning {1:N0} bytes", (DateTime.Now - start).TotalMilliseconds, ret.Length)); } return ret; }
void AQSubscriberTextToAnnotate_OnMessageReceived(string message) { try { lock (_mqLock) { StoreEventData(_activeMQSettings.TopicNameTextToAnnotate, message); IncreaseProcessedCount(); HtmlAgilityPack.XmlDocument xmlDoc = new HtmlAgilityPack.XmlDocument(); xmlDoc.LoadXml(message); _keui.AnnotateTextToAnnotate(xmlDoc); UpdateEventMetaData(xmlDoc, _activeMQSettings.TopicNameKEUIPublishTextToAnnotate); // publish it string annotatedMessage = xmlDoc.DocumentNode.OuterHtml; StoreEventData(_activeMQSettings.TopicNameKEUIPublishTextToAnnotate, annotatedMessage); using (var AQPublisherTextToAnnotate = new ActiveMqHelper.TopicPublisher(AQSession, _activeMQSettings.TopicNameKEUIPublishTextToAnnotate)) { AQPublisherTextToAnnotate.SendMessage(annotatedMessage); } } } catch (Exception ex) { AddEvent("Exception while annotating general text: " + ex.Message); GenLib.Log.LogService.LogException("Exception while annotating general text. message: " + message, ex); } }
public string SetData(string data) { string ret = ""; DateTime start = DateTime.Now; MailData.LogInfo("SetData data: " + data); HtmlAgilityPack.XmlDocument dataDoc = new HtmlAgilityPack.XmlDocument(); dataDoc.LoadXml(data); var dataNode = dataDoc.DocumentNode.SelectSingleNode("/data"); string target = dataNode.GetAttributeValue("target", null); string content = dataNode.GetAttributeValue("content", null); if (target == "MailData") ret = MailData.SetData(content, dataNode); else if (target == "TagData") ret = MailData.SetTagData(content, dataNode); else MailData.LogInfo("Warning: Don't know how to use the data: " + data); MailData.LogInfo(String.Format("SetData needed {0:N0} ms", (DateTime.Now - start).TotalMilliseconds)); return ret; }
void AQSubscriberWikiPostNew_OnMessageReceived(string message) { try { lock (_mqLock) { StoreEventData(_activeMQSettings.TopicNameWikiPostNew, message); IncreaseProcessedCount(); HtmlAgilityPack.XmlDocument xmlDoc = new HtmlAgilityPack.XmlDocument(); xmlDoc.LoadXml(message); _keui.AnnotateWikiPost(xmlDoc); int itemId = _keui.IndexWikiPost(xmlDoc); if (itemId == -1) StoreEventData(_activeMQSettings.TopicNameWikiPostNew, message, true, "invalid_"); UpdateEventMetaData(xmlDoc, _activeMQSettings.TopicNameKEUIPublishWikiPostNew); // publish it string annotatedMessage = xmlDoc.DocumentNode.OuterHtml; StoreEventData(_activeMQSettings.TopicNameKEUIPublishWikiPostNew, annotatedMessage); using (var publisher = new ActiveMqHelper.TopicPublisher(AQSession, _activeMQSettings.TopicNameKEUIPublishWikiPostNew)) { publisher.SendMessage(annotatedMessage); } } } catch (Exception ex) { AddEvent("Exception while processing new wiki post: " + ex.Message); GenLib.Log.LogService.LogException("Exception while processing new wiki post. message: " + message, ex); } }
public AddItemStatus AddItem(string itemInfo, string itemContent) { try { itemInfo = Text.RemoveControlCharacters(itemInfo); itemContent = Text.RemoveControlCharacters(itemContent); if (string.IsNullOrEmpty(itemInfo)) return AddItemStatus.GetInvalidItemStatus("ItemInfo parameter is empty. Unable to add item to the index"); HtmlAgilityPack.XmlDocument itemInfoDoc = new HtmlAgilityPack.XmlDocument(); itemInfoDoc.LoadXml(itemInfo); bool editedItemInfo = false; HtmlNode itemNode = itemInfoDoc.DocumentNode.SelectSingleNode("/item"); if (itemNode.GetAttributeValue("itemId", -1) == -1) { ESQLItemType itemType = (ESQLItemType)itemNode.GetAttributeValue("itemType", 0); string entryId = itemInfoDoc.DocumentNode.SelectSingleNode("/item/entryId").InnerText.DecodeXMLString(); int accountId = itemNode.GetAttributeValue("accountId", -1); SQLAddItem(entryId, itemType, accountId, ESQLItemState.ToBeAdded); int computedItemId = SQLGetItemId(entryId); itemNode.SetAttributeValue("itemId", computedItemId.ToString()); editedItemInfo = true; } if (itemInfoDoc.DocumentNode.SelectNodes("/item/people/person[@nameTrust]") != null) { foreach (var node in itemInfoDoc.DocumentNode.SelectNodes("/item/people/person[@nameTrust]")) { string account = node.GetAttributeValue("account", null); string name = node.GetAttributeValue("name", null); name = name.DecodeXMLString(); account = account.DecodeXMLString(); int accountType = node.GetAttributeValue("accountType", -1); int nameTrust = node.GetAttributeValue("nameTrust", -1); if (name == null || account == null || accountType == -1 || nameTrust == -1) { GenLib.Log.LogService.LogError("Item was not added to the index due to missing information in the people section. Invalid data is: " + node.InnerHtml); return AddItemStatus.GetInvalidItemStatus("Item was not added to the index due to missing information in the people section. Invalid data is: " + node.InnerHtml); } PeopleData.AddNewPersonOrUpdate(name, account, (EAccountType)accountType, (EPersonNameTrust)nameTrust); int accountId = PeopleData.GetAccountId(account, (EAccountType)accountType); node.SetAttributeValue("id", accountId.ToString()); editedItemInfo = true; } } if (editedItemInfo == true) itemInfo = itemInfoDoc.DocumentNode.InnerHtml; int senderId = -1; List<int> recipientIds = new List<int>(); foreach (var node in itemInfoDoc.DocumentNode.SelectNodes("/item/people/person") ?? new HtmlNodeCollection(null)) { if (_senderRoles.Contains(node.GetAttributeValue("role", null))) senderId = node.GetAttributeValue("id", -1); else if (_recipientRoles.Contains(node.GetAttributeValue("role", null))) recipientIds.Add(node.GetAttributeValue("id", -1)); } if (senderId != -1) { PeopleData.IncreaseCountInfo(senderId, recipientIds); } var subjectNode = itemNode.SelectSingleNode("./subject"); string subject = subjectNode != null ? subjectNode.InnerText.DecodeXMLString() : ""; var metaNode = itemNode.SelectSingleNode("./metaData"); string meta = metaNode != null ? metaNode.InnerHtml : ""; if (Settings.StoreToFileAddItemInfo) StoreToFile("itemInfo", itemInfo); if (Settings.StoreToFileAddItemContent) StoreToFile("itemContent", itemContent); AddItemStatus itemStatus = MinerAddItem(itemInfo, itemContent); if (itemStatus.ItemId != -1) { SQLSetItemState(itemStatus.ItemId, ESQLItemState.Indexed); SQLSetItemSubjectBodyMeta(itemStatus.ItemId, subject, itemContent, meta); } else LogInfo("Indexing of the item failed. Error info: " + itemStatus.ErrorMessage); return itemStatus; } catch (Exception ex) { GenLib.Log.LogService.LogException("MailData. AddItem: ", ex); return AddItemStatus.GetInvalidItemStatus("MailData AddItem exception: " + ex.Message, ex.StackTrace); } }