/// <summary>
/// Download content from a url
/// </summary>
/// <param name="step">Step in crawler that contains url to download</param>
/// <returns>Downloaded content</returns>
private PropertyBag Download(CrawlStep step)
{
try
{
IWebDownloader webDownloader = m_DownloaderFactory.GetDownloader();
m_Logger.Verbose("Downloading {0}", step.Uri);
return(webDownloader.Download(step, DownloadMethod.Get));
}
catch (Exception ex)
{
OnDownloadException(ex, step);
}
return(null);
}
public void Process(Crawler crawler, PropertyBag propertyBag)
{
AspectF.Define.
NotNull(crawler, "crawler").
NotNull(propertyBag, "propertyBag");
string content = propertyBag.Text;
if (content.IsNullOrEmpty())
{
return;
}
string contentLookupText = content.Length > MaxPostSize
? content.Substring(0, MaxPostSize).Trim()
: content.Trim();
string encodedRequestUrlFragment =
"http://ajax.googleapis.com/ajax/services/language/detect?v=1.0&q={0}".FormatWith(contentLookupText);
IWebDownloader downloader = m_DownloaderFactory.GetDownloader();
PropertyBag result = downloader.Download(new CrawlStep(new Uri(encodedRequestUrlFragment), 0), DownloadMethod.Get);
using (MemoryStream responseReader = result.GetResponseStream())
using (StreamReader reader = new StreamReader(responseReader))
{
string json = reader.ReadLine();
using (MemoryStream ms = new MemoryStream(Encoding.Unicode.GetBytes(json)))
{
DataContractJsonSerializer ser =
new DataContractJsonSerializer(typeof(LanguageDetector));
LanguageDetector detector = ser.ReadObject(ms) as LanguageDetector;
if (!detector.IsNull())
{
CultureInfo culture = CultureInfo.GetCultureInfo(detector.responseData.language);
propertyBag["Language"].Value = detector.responseData.language;
propertyBag["LanguageCulture"].Value = culture;
}
}
}
}
public async Task InstallAsync(IDownloadApp app)
{
consoleLogger.Info($"Installing '{app.AppId}'");
IDownloader downloader = downloaderFactory.GetDownloader(app.Downloader);
var downloadedFilePath = await downloader.DownloadAsync(app.DownloaderArgs.ToString() !);
var installScript = $"{app.InstallScript} {downloadedFilePath}";
if (app.VerificationScript == null)
{
await powerShell.ExecuteAsync(installScript);
}
else
{
await powerShell.ExecuteAsync(installScript, app.VerificationScript);
}
consoleLogger.Result($"Installed '{app.AppId}'");
}