public void Process(Crawler crawler, PropertyBag propertyBag) { AspectF.Define. NotNull(crawler, "crawler"). NotNull(propertyBag, "propertyBag"); string content = propertyBag.Text; if (content.IsNullOrEmpty()) { return; } string contentLookupText = content.Max(MaxPostSize); string encodedRequestUrlFragment = "http://ajax.googleapis.com/ajax/services/language/detect?v=1.0&q={0}".FormatWith(contentLookupText); m_Logger.Verbose("Google language detection using: {0}", encodedRequestUrlFragment); try { IWebDownloader downloader = NCrawlerModule.Container.Resolve <IWebDownloader>(); PropertyBag result = downloader.Download(new CrawlStep(new Uri(encodedRequestUrlFragment), 0), null, DownloadMethod.GET); if (result.IsNull()) { return; } using (Stream responseReader = result.GetResponse()) using (StreamReader reader = new StreamReader(responseReader)) { string json = reader.ReadLine(); using (MemoryStream ms = new MemoryStream(Encoding.Unicode.GetBytes(json))) { DataContractJsonSerializer ser = new DataContractJsonSerializer(typeof(LanguageDetector)); LanguageDetector detector = ser.ReadObject(ms) as LanguageDetector; if (!detector.IsNull()) { CultureInfo culture = CultureInfo.GetCultureInfo(detector.responseData.language); propertyBag["Language"].Value = detector.responseData.language; propertyBag["LanguageCulture"].Value = culture; } } } } catch (Exception ex) { m_Logger.Error("Error during google language detection, the error was: {0}", ex.ToString()); } }
public void Process(Crawler crawler, PropertyBag propertyBag) { AspectF.Define. NotNull(crawler, "crawler"). NotNull(propertyBag, "propertyBag"); string content = propertyBag.Text; if (content.IsNullOrEmpty()) { return; } string contentLookupText = content.Length > MaxPostSize ? content.Substring(0, MaxPostSize).Trim() : content.Trim(); string encodedRequestUrlFragment = "http://ajax.googleapis.com/ajax/services/language/detect?v=1.0&q={0}".FormatWith(contentLookupText); IWebDownloader downloader = m_DownloaderFactory.GetDownloader(); PropertyBag result = downloader.Download(new CrawlStep(new Uri(encodedRequestUrlFragment), 0), DownloadMethod.Get); using (MemoryStream responseReader = result.GetResponseStream()) using (StreamReader reader = new StreamReader(responseReader)) { string json = reader.ReadLine(); using (MemoryStream ms = new MemoryStream(Encoding.Unicode.GetBytes(json))) { DataContractJsonSerializer ser = new DataContractJsonSerializer(typeof(LanguageDetector)); LanguageDetector detector = ser.ReadObject(ms) as LanguageDetector; if (!detector.IsNull()) { CultureInfo culture = CultureInfo.GetCultureInfo(detector.responseData.language); propertyBag["Language"].Value = detector.responseData.language; propertyBag["LanguageCulture"].Value = culture; } } } }