public void AutoEncodingTest() { WebClient wc = new WebClient(); var data = wc.HttpGetByte("http://kakaku.com/kaden/"); System.Text.Encoding encoding; var html = wc.GetAutoEncoding(data, out encoding); Trace.WriteLine(encoding); Console.WriteLine(encoding.BodyName); Console.WriteLine(html); }
/// <summary> /// urlとHtmlを入力として、結果を返す。 /// </summary> /// <param name="url"></param> /// <param name="EnableGetSubUrlLink"></param> /// <param name="takeNum"></param> /// <param name="maxPage"></param> /// <returns></returns> public static ResultData GetDataList(string url, bool EnableGetSubUrlLink, int takeNum, int maxPage) { ReportStock rs = new ReportStock(); WebClient wc = new WebClient() { AddUserAgent = true }; rs.Add(wc); var rawler = wc.Page(url).AddRange( new RawlerAutoListDataWrite() { TakeNum = takeNum, EnableGetSubUrlLink = EnableGetSubUrlLink }, new RawlerAutoNextLink() { MaxCount = maxPage }, new RawlerAutoTable()).GetRoot(); rawler.Run(); return new ResultData() { Data = rawler.GetDescendantRawler().OfType<Data>().DataSort().ToArray(), Reports = rs.ReportList }; }
public static ResultData GetDataSinglePages(IEnumerable<string> list) { ReportStock rs = new ReportStock(); WebClient wc = new WebClient() { AddUserAgent = true }; rs.Add(wc); var rawler = wc.Add(new Data()).Add(new RawlerAutoSingelPage() { SampleUrls = list.ToList() }).GetRoot(); rawler.Run(); var xaml = rawler.ToXAML(); return new ResultData() { Data = rawler.GetDescendantRawler().OfType<Data>(), Reports = rs.ReportList }; }
public static IEnumerable<HTMLAnalyzeResult> GetListHTMLStruct(Uri uri, bool EnableGetSubUrlLink) { WebClient wc = new WebClient() { AddUserAgent = true }; return GetListHTMLStruct(wc.HttpGet(uri.AbsoluteUri), uri.AbsolutePath, EnableGetSubUrlLink); }
public static Rawler.Tool.RawlerBase GetDataListRawlerBase(string url, bool EnableGetSubUrlLink, int takeNum, int maxPage) { ReportStock rs = new ReportStock(); WebClient wc = new WebClient() { AddUserAgent = true }; rs.Add(wc); return wc.Page(url).Add(new RawlerAutoListDataWrite() { TakeNum = takeNum, EnableGetSubUrlLink = EnableGetSubUrlLink }).Add(new RawlerAutoNextLink() { MaxCount = maxPage }).GetRoot(); }