public void Process(Crawler crawler, PropertyBag propertyBag)
{
AspectF.Define.
NotNull(crawler, "crawler").
NotNull(propertyBag, "propertyBag");
string content = propertyBag.Text;
if (content.IsNullOrEmpty())
{
return;
}
string contentLookupText = content.Max(MaxPostSize);
string encodedRequestUrlFragment =
"http://ajax.googleapis.com/ajax/services/language/detect?v=1.0&q={0}".FormatWith(contentLookupText);
m_Logger.Verbose("Google language detection using: {0}", encodedRequestUrlFragment);
try
{
IWebDownloader downloader = NCrawlerModule.Container.Resolve <IWebDownloader>();
PropertyBag result = downloader.Download(new CrawlStep(new Uri(encodedRequestUrlFragment), 0), null, DownloadMethod.GET);
if (result.IsNull())
{
return;
}
using (Stream responseReader = result.GetResponse())
using (StreamReader reader = new StreamReader(responseReader))
{
string json = reader.ReadLine();
using (MemoryStream ms = new MemoryStream(Encoding.Unicode.GetBytes(json)))
{
DataContractJsonSerializer ser =
new DataContractJsonSerializer(typeof(LanguageDetector));
LanguageDetector detector = ser.ReadObject(ms) as LanguageDetector;
if (!detector.IsNull())
{
CultureInfo culture = CultureInfo.GetCultureInfo(detector.responseData.language);
propertyBag["Language"].Value = detector.responseData.language;
propertyBag["LanguageCulture"].Value = culture;
}
}
}
}
catch (Exception ex)
{
m_Logger.Error("Error during google language detection, the error was: {0}", ex.ToString());
}
}
public void Process(Crawler crawler, PropertyBag propertyBag)
{
AspectF.Define.
NotNull(crawler, "crawler").
NotNull(propertyBag, "propertyBag");
string content = propertyBag.Text;
if (content.IsNullOrEmpty())
{
return;
}
string contentLookupText = content.Length > MaxPostSize
? content.Substring(0, MaxPostSize).Trim()
: content.Trim();
string encodedRequestUrlFragment =
"http://ajax.googleapis.com/ajax/services/language/detect?v=1.0&q={0}".FormatWith(contentLookupText);
IWebDownloader downloader = m_DownloaderFactory.GetDownloader();
PropertyBag result = downloader.Download(new CrawlStep(new Uri(encodedRequestUrlFragment), 0), DownloadMethod.Get);
using (MemoryStream responseReader = result.GetResponseStream())
using (StreamReader reader = new StreamReader(responseReader))
{
string json = reader.ReadLine();
using (MemoryStream ms = new MemoryStream(Encoding.Unicode.GetBytes(json)))
{
DataContractJsonSerializer ser =
new DataContractJsonSerializer(typeof(LanguageDetector));
LanguageDetector detector = ser.ReadObject(ms) as LanguageDetector;
if (!detector.IsNull())
{
CultureInfo culture = CultureInfo.GetCultureInfo(detector.responseData.language);
propertyBag["Language"].Value = detector.responseData.language;
propertyBag["LanguageCulture"].Value = culture;
}
}
}
}
