private void button1_Click(object sender, EventArgs e)
{
HrefList = HtmlFunction.GetPageHref("C:\\Users\\wjing\\Desktop\\test.txt");
this.listView1.Columns.Add("Index", 50);
this.listView1.Columns.Add("Urllist", 800);
for (int i = 0; i < HrefList.Count; i++) //添加10行数据
{
ListViewItem Row = new ListViewItem(i.ToString()); //这个是第一行第一列
this.listView1.Items.Add(Row);
listView1.Items[i].SubItems.Add(HrefList[i].ToString());
}
this.listView1.EndUpdate(); //结束数据处理，UI界面一次性绘制。
}
public static void DownloadFile(object o)
{
CrawlerPara cp = (CrawlerPara)o;
Encoding encoding = null;
try
{
//获取并设置编码方式
string Chara = HtmlFunction.GetEncoding(cp.sUrl);
if (Chara == "GBK" || Chara == "gbk")
{
encoding = Encoding.GetEncoding("GBK");
}
else if (Chara == "GB2312" || Chara == "gb2312")
{
encoding = Encoding.GetEncoding("gb2312");
}
else if (Chara == "UTF-8" || Chara == "utf8" || Chara == "utf-8")
{
encoding = Encoding.GetEncoding("UTF-8");
}
//发送请求
HttpWebRequest mClient = (HttpWebRequest)WebRequest.Create(cp.sUrl);
HttpWebResponse mResp = (HttpWebResponse)mClient.GetResponse();
//创建读写器
StreamReader mReader;
if (mResp.ContentEncoding != null && mResp.ContentEncoding.Equals("gzip", StringComparison.InvariantCultureIgnoreCase))
{
mReader = new StreamReader(new GZipStream(mResp.GetResponseStream(), CompressionMode.Decompress), encoding);
}
else
{
mReader = new StreamReader(mResp.GetResponseStream(), encoding);
}
//读取数据并写入到文件
string sline;
FileStream htmlfile = new FileStream(cp.sfilepath, FileMode.Create);
StreamWriter sw = new StreamWriter(htmlfile, Encoding.Unicode);
sline = mReader.ReadLine();
while (mReader.EndOfStream == false)
{
string strtemp = "";
if (Chara == "GBK" || Chara == "gbk")
{
strtemp = HtmlFunction.Gbk_To_Unicode(sline);
}
else if (Chara == "GB2312" || Chara == "gb2312")
{
strtemp = HtmlFunction.Gb2312_To_Unicode(sline);
}
else if (Chara == "UTF-8" || Chara == "utf8" || Chara == "utf-8")
{
strtemp = HtmlFunction.Utf8_To_Unicode(sline);
}
sw.WriteLine(strtemp);
sline = mReader.ReadLine();
}
mReader.Close();
sw.Close();
}
catch (Exception e) {}
//委托调用返回处理结果
cp.callMothd(cp.sUrl);
}
