Пример #1
0
        /// <summary>
        /// Download content from a url
        /// </summary>
        /// <param name="step">Step in crawler that contains url to download</param>
        /// <returns>Downloaded content</returns>
        private PropertyBag Download(CrawlStep step)
        {
            try
            {
                IWebDownloader webDownloader = m_DownloaderFactory.GetDownloader();
                m_Logger.Verbose("Downloading {0}", step.Uri);
                return(webDownloader.Download(step, DownloadMethod.Get));
            }
            catch (Exception ex)
            {
                OnDownloadException(ex, step);
            }

            return(null);
        }
Пример #2
0
        public void Process(Crawler crawler, PropertyBag propertyBag)
        {
            AspectF.Define.
            NotNull(crawler, "crawler").
            NotNull(propertyBag, "propertyBag");

            string content = propertyBag.Text;

            if (content.IsNullOrEmpty())
            {
                return;
            }

            string contentLookupText = content.Length > MaxPostSize
                                ? content.Substring(0, MaxPostSize).Trim()
                                : content.Trim();

            string encodedRequestUrlFragment =
                "http://ajax.googleapis.com/ajax/services/language/detect?v=1.0&q={0}".FormatWith(contentLookupText);

            IWebDownloader downloader = m_DownloaderFactory.GetDownloader();
            PropertyBag    result     = downloader.Download(new CrawlStep(new Uri(encodedRequestUrlFragment), 0), DownloadMethod.Get);

            using (MemoryStream responseReader = result.GetResponseStream())
                using (StreamReader reader = new StreamReader(responseReader))
                {
                    string json = reader.ReadLine();
                    using (MemoryStream ms = new MemoryStream(Encoding.Unicode.GetBytes(json)))
                    {
                        DataContractJsonSerializer ser =
                            new DataContractJsonSerializer(typeof(LanguageDetector));
                        LanguageDetector detector = ser.ReadObject(ms) as LanguageDetector;

                        if (!detector.IsNull())
                        {
                            CultureInfo culture = CultureInfo.GetCultureInfo(detector.responseData.language);
                            propertyBag["Language"].Value        = detector.responseData.language;
                            propertyBag["LanguageCulture"].Value = culture;
                        }
                    }
                }
        }
        public async Task InstallAsync(IDownloadApp app)
        {
            consoleLogger.Info($"Installing '{app.AppId}'");

            IDownloader downloader = downloaderFactory.GetDownloader(app.Downloader);

            var downloadedFilePath = await downloader.DownloadAsync(app.DownloaderArgs.ToString() !);

            var installScript = $"{app.InstallScript} {downloadedFilePath}";

            if (app.VerificationScript == null)
            {
                await powerShell.ExecuteAsync(installScript);
            }
            else
            {
                await powerShell.ExecuteAsync(installScript, app.VerificationScript);
            }

            consoleLogger.Result($"Installed '{app.AppId}'");
        }