public void AutoEncodingTest()
{
WebClient wc = new WebClient();
var data = wc.HttpGetByte("http://kakaku.com/kaden/");
System.Text.Encoding encoding;
var html = wc.GetAutoEncoding(data, out encoding);
Trace.WriteLine(encoding);
Console.WriteLine(encoding.BodyName);
Console.WriteLine(html);
}
/// <summary>
/// urlとHtmlを入力として、結果を返す。
/// </summary>
/// <param name="url"></param>
/// <param name="EnableGetSubUrlLink"></param>
/// <param name="takeNum"></param>
/// <param name="maxPage"></param>
/// <returns></returns>
public static ResultData GetDataList(string url, bool EnableGetSubUrlLink, int takeNum, int maxPage)
{
ReportStock rs = new ReportStock();
WebClient wc = new WebClient() { AddUserAgent = true };
rs.Add(wc);
var rawler = wc.Page(url).AddRange(
new RawlerAutoListDataWrite() { TakeNum = takeNum, EnableGetSubUrlLink = EnableGetSubUrlLink },
new RawlerAutoNextLink() { MaxCount = maxPage },
new RawlerAutoTable()).GetRoot();
rawler.Run();
return new ResultData() { Data = rawler.GetDescendantRawler().OfType<Data>().DataSort().ToArray(), Reports = rs.ReportList };
}
public static ResultData GetDataSinglePages(IEnumerable<string> list)
{
ReportStock rs = new ReportStock();
WebClient wc = new WebClient() { AddUserAgent = true };
rs.Add(wc);
var rawler = wc.Add(new Data()).Add(new RawlerAutoSingelPage() { SampleUrls = list.ToList() }).GetRoot();
rawler.Run();
var xaml = rawler.ToXAML();
return new ResultData() { Data = rawler.GetDescendantRawler().OfType<Data>(), Reports = rs.ReportList };
}
public static IEnumerable<HTMLAnalyzeResult> GetListHTMLStruct(Uri uri, bool EnableGetSubUrlLink)
{
WebClient wc = new WebClient() { AddUserAgent = true };
return GetListHTMLStruct(wc.HttpGet(uri.AbsoluteUri), uri.AbsolutePath, EnableGetSubUrlLink);
}
public static Rawler.Tool.RawlerBase GetDataListRawlerBase(string url, bool EnableGetSubUrlLink, int takeNum, int maxPage)
{
ReportStock rs = new ReportStock();
WebClient wc = new WebClient() { AddUserAgent = true };
rs.Add(wc);
return wc.Page(url).Add(new RawlerAutoListDataWrite() { TakeNum = takeNum, EnableGetSubUrlLink = EnableGetSubUrlLink }).Add(new RawlerAutoNextLink() { MaxCount = maxPage }).GetRoot();
}
