public SearchResultList Search(SearchContext searchContext)
{
SearchResultList resultList = null;
if (hasInsufficientBalance)
{
LogService.Warn(this.GetType(),"Insufficient blance for Bing search request");
}
else
{
switch (searchContext.SearchType)
{
case SearchTypeEnum.News:
resultList = SearchNews(searchContext);
break;
case SearchTypeEnum.Web:
resultList = SearchWeb(searchContext);
break;
case SearchTypeEnum.Image:
resultList = SearchImages(searchContext);
break;
default:
resultList = new SearchResultList();
break;
}
}
return resultList;
}
public void AddResults(SearchResultList resultList)
{
foreach (SearchResultItem newResult in resultList)
{
SearchResultItem result = GetResult(newResult.URL);
if (result == null)
{
Add(newResult);
}
}
}
public void Parse(JObject searchResultsJSONObject, SearchResultList searchResultList)
{
try
{
JArray resultsJArray = searchResultsJSONObject.Value<JArray>("statuses");
foreach (JObject resultJObject in resultsJArray)
{
SearchResultItem resultItem = new SearchResultItem();
resultItem.Provider = ProviderEnum.Twitter;
string content = resultJObject.Value<string>("text");
if (content != null && !content.Contains("#"))
{
resultItem.Abstract = content.ParseTwitterURL().ParseTwitterUsername().ParseTwitterHashtag();
JObject userDataJObject = resultJObject.Value<JObject>("user");
string image = userDataJObject.Value<string>("profile_image_url");
resultItem.ImageURL = image;
string name = userDataJObject.Value<string>("name");
resultItem.Title = name;
string screenName = userDataJObject.Value<string>("screen_name");
string url = "http://www.twitter.com/" + screenName;
resultItem.URL = url;
string publishedDateValue = resultJObject.Value<string>("created_at");
if (publishedDateValue != null)
{
// format: Sun, 30 Oct 2011 19:54:11 +0000",(publishedDateValue, "ddd, dd MMM yyyy HH':'mm':'ss z");
// new format Mon Sep 24 03:35:21 +0000 2012
//publishedDateValue = publishedDateValue.Replace("+0000", "+0");
//DateTime publishedDate = DateParser.Parse(publishedDateValue, "ddd MMM HH':'mm':'ss z yyyy");
//resultItem.PublishedDate = publishedDate;
}
searchResultList.Add(resultItem);
}
}
}
catch (Exception exception)
{
ErrorService.Log("TwitterSearchParser", "Search", searchResultList.ToString(), exception);
}
}
// INCORRECT RESULT
//{
// "responseData": null,
// "responseDetails": "Quota Exceeded. Please see http://code.google.com/apis/websearch",
// "responseStatus": 403
//}
// CORRECT RESULT
// {
// "responseData": {
// "results": [
// {
// "GsearchResultClass": "GwebSearch",
// "unescapedUrl": "http://en.wikipedia.org/wiki/Stefan_Savi%C4%87",
// "url": "http://en.wikipedia.org/wiki/Stefan_Savi%25C4%2587",
// "visibleUrl": "en.wikipedia.org",
// "cacheUrl": "http://www.google.com/search?q=cache:HWRCbR1U9oEJ:en.wikipedia.org",
// "title": "<b>Stefan Savi?</b> - Wikipedia, the free encyclopedia",
// "titleNoFormatting": "Stefan Savi? - Wikipedia, the free encyclopedia",
// "content": "<b>Stefan Savi?</b> (Cyrillic: ?????? ?????; born 8 January 1991) is a Montenegrin footballer who plays for Manchester City in the Premier League. A centre-back <b>...</b>"
// }
// ],
// "cursor": {
// "resultCount": "55,500",
// "pages": [
// {
// "start": "0",
// "label": 1
// }
// ],
// "estimatedResultCount": "55500",
// "currentPageIndex": 0,
// "moreResultsUrl": "http://www.google.com/search?oe=utf8&ie=utf8&source=uds&start=0&hl=en&q=Stefan+Savic",
// "searchResultTime": "0.18"
// }
// },
// "responseDetails": null,
// "responseStatus": 200
//}
public void Parse(JObject searchResultsJSONObject, SearchResultList searchResultList)
{
try
{
JObject responseDataJObject = searchResultsJSONObject.Value<JObject>("responseData");
if (responseDataJObject != null)
{
JArray resultsJArray = responseDataJObject.Value<JArray>("results");
foreach (JObject resultJObject in resultsJArray)
{
SearchResultItem resultItem = new SearchResultItem();
resultItem.Provider = ProviderEnum.Google;
string title = resultJObject.Value<string>("titleNoFormatting");
resultItem.Title = TextCleaner.RemoveHtml(title);
string content = resultJObject.Value<string>("content");
resultItem.Abstract = TextCleaner.RemoveHtml(content);
string url = resultJObject.Value<string>("unescapedUrl");
resultItem.URL = TextCleaner.RemoveHtml(url);
string publisher = resultJObject.Value<string>("publisher");
if (publisher != null)
{
resultItem.Source = publisher;
}
string publishedDateValue = resultJObject.Value<string>("publishedDate");
if (publishedDateValue != null)
{
// format: Thu, 19 Nov 2009 13:05:26 -08:00
DateTime publishedDate = DateParser.Parse(publishedDateValue, "ddd, dd MMM yyyy HH':'mm':'ss zzz");
resultItem.PublishedDate = publishedDate;
}
searchResultList.Add(resultItem);
}
}
}
catch (Exception exception)
{
ErrorService.Log("GoogleSearchParser", "Search", searchResultList.ToString(), exception);
}
}
private SearchResultList SearchNews(SearchContext searchContext)
{
SearchResultList resultList = new SearchResultList();
try
{
string query = searchContext.Query;
//int page = searchContext.Page;
//int count = searchContext.Count;
//int start = (page - 1) * count;
string accountKey = ConfigService.GetConfig(ConfigKeys.BING_API_KEY, "");
BingSearchContainer bingContainer = new BingSearchContainer(new Uri("https://api.datamarket.azure.com/Bing/Search/News"));
bingContainer.Credentials = new NetworkCredential(accountKey, accountKey);
DataServiceQuery<NewsResult> newsQuery = bingContainer.News(query, "en-us", "strict", null, null, null, null, null);
IEnumerable<NewsResult> newsResults = newsQuery.Execute();
foreach (NewsResult result in newsResults)
{
SearchResultItem searchResultItem = new SearchResultItem();
searchResultItem.Provider = ProviderEnum.Bing;
searchResultItem.Title = result.Title;
searchResultItem.URL = result.Url;
searchResultItem.Source = result.Source;
searchResultItem.PublishedDate = (DateTime) result.Date;
string abstractText = result.Description;
abstractText = TextCleaner.StripTag(abstractText, "<", ">"); // remove any html tags
abstractText = TextCleaner.RemoveHtml(abstractText);
searchResultItem.Abstract = abstractText;
resultList.Add(searchResultItem);
}
}
catch (Exception exception)
{
ErrorService.Log("BingSearchService", "SearchNews", searchContext.ToString(), exception);
if (exception.ToString().Contains("Insufficient balance"))
{
hasInsufficientBalance = true;
}
}
return resultList;
}
public SearchResultList Search(SearchContext searchContext)
{
SearchResultList resultList = new SearchResultList();
try
{
string urlTemplate;
switch (searchContext.SearchType)
{
case SearchTypeEnum.News:
urlTemplate = "http://ajax.googleapis.com/ajax/services/search/news?v={0}&rsz=large&key={1}&q={2}&start={3}";
break;
case SearchTypeEnum.Web:
urlTemplate = "http://ajax.googleapis.com/ajax/services/search/web?v={0}&rsz=large&key={1}&q={2}&start={3}";
break;
default:
return resultList;
}
string api = ConfigService.GetConfig(ConfigKeys.GOOGLE_API_KEY, "");
string query = searchContext.Query;
int countPerPage = ConfigService.GetConfig(ConfigKeys.THEINTERNETBUZZ_SEARCH_COUNT_PER_PAGE_PER_PROVIDER, 8);
int page = (searchContext.Page - 1) * countPerPage;
string url = string.Format(urlTemplate, API_VERSION, api, HttpUtility.UrlEncode(query), page);
JSONConnector JSONConnector = new JSONConnector();
JObject searchResultsJSONObject = JSONConnector.GetJSONObject(url);
new GoogleSearchParser().Parse(searchResultsJSONObject, resultList);
}
catch (Exception exception)
{
ErrorService.Log("GoogleSearchService", "Search", searchContext.ToString(), exception);
}
return resultList;
}
public SearchResultList Search(SearchContext searchContext)
{
SearchResultList resultList = new SearchResultList();
try
{
string urlTemplate;
switch (searchContext.SearchType)
{
case SearchTypeEnum.Tweet:
urlTemplate = "https://api.twitter.com/1.1/search/tweets.json?q={0}&count={1}&result_type=mixed&include_entities=false";
break;
default:
return resultList;
}
TwitterCredentials accessToken = new TwitterAuthService().Authenticate();
string headerName = "Authorization";
string headerValue = accessToken.User + " " + accessToken.Token;
string query = searchContext.Query;
int countPerPage = ConfigService.GetConfig(ConfigKeys.THEINTERNETBUZZ_SEARCH_COUNT_PER_PAGE_PER_PROVIDER, 8);
int page = searchContext.Page;
string url = string.Format(urlTemplate, HttpUtility.UrlEncode(query), countPerPage);
JSONConnector JSONConnector = new JSONConnector();
JObject searchResultsJSONObject = JSONConnector.GetJSONObjectWithHeader(url, headerName, headerValue);
new TwitterSearchParser().Parse(searchResultsJSONObject, resultList);
}
catch (Exception exception)
{
ErrorService.Log("TwitterSearchService", "Search", searchContext.ToString(), exception);
}
return resultList;
}
public SearchResultList Search(SearchContext searchContext)
{
SearchResultList resultList = new SearchResultList();
try
{
string urlTemplate;
switch (searchContext.SearchType)
{
case SearchTypeEnum.Image:
urlTemplate = "http://api.flickr.com/services/rest/?method=flickr.photos.search&api_key={0}&safe_search=1&per_page={1}&page={2}&format=rest&nojsoncallback=1&privacy_filter=1&content_type=1&sort=relevance&text={3}";
break;
default:
return resultList;
}
string query = searchContext.Query;
int countPerPage = searchContext.Count;
int page = searchContext.Page;
string key = ConfigService.GetConfig(ConfigKeys.FLICKR_API_KEY, "");
string url = string.Format(urlTemplate, key, countPerPage, page, HttpUtility.UrlEncode(query));
Console.WriteLine(url);
XMLConnector XMLConnector = new XMLConnector();
XmlDocument xmlDocument = XMLConnector.GetXMLDocument(url);
new FlickrSearchParser().Parse(xmlDocument, resultList);
}
catch (Exception exception)
{
ErrorService.Log("FlickrSearchService", "Search", searchContext.ToString(), exception);
}
return resultList;
}
public void Parse(XmlDocument xmlDocument, SearchResultList searchResultList)
{
string urlTemplate = "http://farm{0}.staticflickr.com/{1}/{2}_{3}.jpg";
try
{
XmlNodeList resultNodes = xmlDocument.SelectNodes("rsp/photos/photo");
if (resultNodes != null)
{
foreach (XmlNode resultNode in resultNodes)
{
Console.WriteLine("resultNode");
SearchResultItem resultItem = new SearchResultItem();
resultItem.Provider = ProviderEnum.Flickr;
string id = resultNode.Attributes["id"].Value;
string secret = resultNode.Attributes["secret"].Value;
string server = resultNode.Attributes["server"].Value;
string farm = resultNode.Attributes["farm"].Value;
string title = resultNode.Attributes["title"].Value;
string url = string.Format(urlTemplate, farm, server, id, secret);
resultItem.ImageURL = url;
resultItem.Title = title;
Console.WriteLine(url);
searchResultList.Add(resultItem);
}
}
}
catch (Exception exception)
{
ErrorService.Log("FlickrSearchParser", "Search", searchResultList.ToString(), exception);
}
}
private SearchResultList SearchNews(SearchContext searchContext)
{
SearchResultList resultList = new SearchResultList();
try
{
string query = searchContext.Query;
int page = searchContext.Page;
int count = searchContext.Count;
int start = (page - 1) * count;
string urlTemplate = "http://boss.yahooapis.com/ysearch/news/v1/{0}?appid={1}&start={2}&count={3}&orderby=date&age=7d&style=raw&format=xml";
string api = ConfigService.GetConfig(ConfigKeys.YAHOO_API_KEY, "");
string url = string.Format(urlTemplate, query, api, start, count);
XmlDocument xmlDocument = new XMLConnector().GetXMLDocument(url);
XmlNamespaceManager nsmanager = new XmlNamespaceManager(xmlDocument.NameTable);
nsmanager.AddNamespace("default", "http://www.inktomi.com/");
XmlNodeList resultNodes = xmlDocument.SelectNodes("default:ysearchresponse/default:resultset_news/default:result", nsmanager);
if (resultNodes != null)
{
foreach (XmlNode resultNode in resultNodes)
{
SearchResultItem resultItem = new SearchResultItem();
resultItem.Provider = ProviderEnum.Yahoo;
XmlNode titleNode = resultNode.SelectSingleNode("default:title", nsmanager);
if (titleNode != null)
{
resultItem.Title = titleNode.InnerText;
}
XmlNode abstractNode = resultNode.SelectSingleNode("default:abstract", nsmanager);
if (abstractNode != null)
{
string abstractText = abstractNode.InnerText;
abstractText = TextCleaner.StripTag(abstractText, "<", ">"); // remove any html tags
abstractText = TextCleaner.RemoveHtml(abstractText);
resultItem.Abstract = abstractText;
}
XmlNode urlNode = resultNode.SelectSingleNode("default:url", nsmanager);
if (urlNode != null)
{
resultItem.URL = urlNode.InnerText;
}
XmlNode sourceNode = resultNode.SelectSingleNode("default:source", nsmanager);
if (sourceNode != null)
{
resultItem.Source = sourceNode.InnerText;
}
XmlNode dateNode = resultNode.SelectSingleNode("default:date", nsmanager);
XmlNode timeNode = resultNode.SelectSingleNode("default:time", nsmanager);
if (dateNode != null && timeNode != null)
{
//format <date>2009/11/20</date><time>01:03:57</time>
string dateString = dateNode.InnerText + " " + timeNode.InnerText + " +0";
if (dateString != null)
{
DateTime publishedDate = DateParser.Parse(dateString, "yyyy'/'MM'/'dd HH':'mm':'ss z");
resultItem.PublishedDate = publishedDate;
}
}
resultList.Add(resultItem);
}
}
}
catch (Exception exception)
{
ErrorService.Log("YahooSearchService", "SearchNews", searchContext.ToString(), exception);
}
return resultList;
}
public static void CacheSearchResultList(SearchTypeEnum searchType, string query, int page, SearchResultList searchResultList)
{
string cacheName = "search-" + searchType + "-" + query + "-" + page;
int cacheExpiration = ConfigService.GetConfig(ConfigKeys.THEINTERNETBUZZ_SEARCH_EXPIRATION, 60);
CacheService.Put(cacheName, cacheExpiration, searchResultList);
}
public SearchResultList Search(SearchContext searchContext)
{
AuditServiceItem auditServiceItem = AuditService.Register("SearchService", "Search", searchContext.ToString());
AuditService.Start(auditServiceItem);
SearchTypeEnum searchType = searchContext.SearchType;
string query = searchContext.Query;
int page = searchContext.Page;
int count = searchContext.Count;
SearchResultList resultList = SearchCacheHelper.ReadSearchResultList(searchType, query, page);
if (resultList == null)
{
resultList = new SearchResultList();
try
{
if (searchType == SearchTypeEnum.Web || searchType == SearchTypeEnum.News || searchType == SearchTypeEnum.Tweet)
{
ManualResetEvent[] doneEvents = new ManualResetEvent[1];
//doneEvents[0] = new ManualResetEvent(false);
//SearchContext bingSearchContext = new SearchContext(ProviderEnum.Bing, searchType, query, page, count);
//SearchWorkerThread bingWorkerThread = new SearchWorkerThread(bingSearchContext, doneEvents[0]);
//ThreadPool.QueueUserWorkItem(bingWorkerThread.ThreadPoolCallback);
doneEvents[0] = new ManualResetEvent(false);
SearchContext twitterSearchContext = new SearchContext(ProviderEnum.Twitter, searchType, query, page, count);
SearchWorkerThread twitterWorkerThread = new SearchWorkerThread(twitterSearchContext, doneEvents[0]);
ThreadPool.QueueUserWorkItem(twitterWorkerThread.ThreadPoolCallback);
//doneEvents[1] = new ManualResetEvent(false);
//SearchContext googleSearchContext = new SearchContext(ProviderEnum.Google, searchType, query, page, count);
//SearchWorkerThread googleWorkerThread = new SearchWorkerThread(googleSearchContext, doneEvents[1]);
//ThreadPool.QueueUserWorkItem(googleWorkerThread.ThreadPoolCallback);
//doneEvents[2] = new ManualResetEvent(false);
//SearchContext yahooSearchContext = new SearchContext(ProviderEnum.Yahoo, searchType, query, page, count);
//SearchWorkerThread yahooWorkerThread = new SearchWorkerThread(yahooSearchContext, doneEvents[2]);
//ThreadPool.QueueUserWorkItem(yahooWorkerThread.ThreadPoolCallback);
WaitHandle.WaitAll(doneEvents);
//if (googleWorkerThread.SearchResultList != null) resultList.AddResults(googleWorkerThread.SearchResultList);
//if (bingWorkerThread.SearchResultList != null) resultList.AddResults(bingWorkerThread.SearchResultList);
//if (yahooWorkerThread.SearchResultList != null) resultList.AddResults(yahooWorkerThread.SearchResultList);
if (twitterWorkerThread.SearchResultList != null) resultList.AddResults(twitterWorkerThread.SearchResultList);
if (searchType == SearchTypeEnum.News)
{
FilterNewsByTitle(resultList, query);
}
}
else if (searchType == SearchTypeEnum.Image)
{
ManualResetEvent[] doneEvents = new ManualResetEvent[1];
doneEvents[0] = new ManualResetEvent(false);
SearchContext flickrSearchContext = new SearchContext(ProviderEnum.Flickr, searchType, query, page, count);
SearchWorkerThread flickrWorkerThread = new SearchWorkerThread(flickrSearchContext, doneEvents[0]);
ThreadPool.QueueUserWorkItem(flickrWorkerThread.ThreadPoolCallback);
//doneEvents[0] = new ManualResetEvent(false);
//SearchContext bingSearchContext = new SearchContext(ProviderEnum.Bing, searchType, query, page, count);
//SearchWorkerThread bingWorkerThread = new SearchWorkerThread(bingSearchContext, doneEvents[0]);
//ThreadPool.QueueUserWorkItem(bingWorkerThread.ThreadPoolCallback);
//doneEvents[0] = new ManualResetEvent(false);
//SearchContext twitterSearchContext = new SearchContext(ProviderEnum.Twitter, searchType, query, page, count);
//SearchWorkerThread twitterWorkerThread = new SearchWorkerThread(twitterSearchContext, doneEvents[0]);
//ThreadPool.QueueUserWorkItem(twitterWorkerThread.ThreadPoolCallback);
//doneEvents[1] = new ManualResetEvent(false);
//SearchContext googleSearchContext = new SearchContext(ProviderEnum.Google, searchType, query, page, count);
//SearchWorkerThread googleWorkerThread = new SearchWorkerThread(googleSearchContext, doneEvents[1]);
//ThreadPool.QueueUserWorkItem(googleWorkerThread.ThreadPoolCallback);
//doneEvents[2] = new ManualResetEvent(false);
//SearchContext yahooSearchContext = new SearchContext(ProviderEnum.Yahoo, searchType, query, page, count);
//SearchWorkerThread yahooWorkerThread = new SearchWorkerThread(yahooSearchContext, doneEvents[2]);
//ThreadPool.QueueUserWorkItem(yahooWorkerThread.ThreadPoolCallback);
WaitHandle.WaitAll(doneEvents);
//if (googleWorkerThread.SearchResultList != null) resultList.AddResults(googleWorkerThread.SearchResultList);
if (flickrWorkerThread.SearchResultList != null) resultList.AddResults(flickrWorkerThread.SearchResultList);
//if (yahooWorkerThread.SearchResultList != null) resultList.AddResults(yahooWorkerThread.SearchResultList);
//if (twitterWorkerThread.SearchResultList != null) resultList.AddResults(twitterWorkerThread.SearchResultList);
}
SearchCacheHelper.CacheSearchResultList(searchType, query, page, resultList);
}
catch (Exception exception)
{
ErrorService.Log("SearchService", "Search", searchContext.ToString(), exception);
return null;
}
}
AuditService.End(auditServiceItem);
return resultList;
}
private void FilterNewsByTitle(SearchResultList resultList, String query)
{
SearchResultList removalList = new SearchResultList();
foreach (SearchResultItem resultItem in resultList)
{
String title = resultItem.Title;
if (title == null || (!title.Contains(query)))
{
removalList.Add(resultItem);
}
}
foreach (SearchResultItem removalItem in removalList)
{
resultList.Remove(removalItem);
}
}
private SearchResultList SearchImages(SearchContext searchContext)
{
SearchResultList resultList = new SearchResultList();
try
{
string query = searchContext.Query;
//int page = searchContext.Page;
//int count = searchContext.Count;
//int start = (page - 1) * count;
string accountKey = ConfigService.GetConfig(ConfigKeys.BING_API_KEY, "");
BingSearchContainer bingContainer = new BingSearchContainer(new Uri("https://api.datamarket.azure.com/Bing/Search/Images/"));
bingContainer.Credentials = new NetworkCredential(accountKey, accountKey);
DataServiceQuery<ImageResult> webQuery = bingContainer.Image(query, "en-us", "strict", null, null, null);
IEnumerable<ImageResult> imageResults = webQuery.Execute();
foreach (ImageResult result in imageResults)
{
SearchResultItem searchResultItem = new SearchResultItem();
searchResultItem.Provider = ProviderEnum.Bing;
searchResultItem.Title = result.Title;
searchResultItem.URL = result.MediaUrl;
searchResultItem.ImageURL = result.MediaUrl;
searchResultItem.Abstract = result.Title;
resultList.Add(searchResultItem);
}
}
catch (Exception exception)
{
ErrorService.Log("BingSearchService", "SearchImages", searchContext.ToString(), exception);
if (exception.ToString().Contains("Insufficient balance"))
{
hasInsufficientBalance = true;
}
}
return resultList;
}