public static string ReadFileInfo(string fileName, string eccoding = "utf8")
{
if (!File.Exists(fileName))
{
return("");
}
Encoding cdg = eccoding == "utf8" ? Encoding.UTF8 : Encoding.Default;
try
{
using (StreamReader sr = new StreamReader(fileName, cdg))
{
string result = sr.ReadToEnd();
sr.Close();
sr.Dispose();
return(result);
}
}
catch (Exception ex)
{
LogServer.WriteLog(ex);
return("");
}
}
public static string DoloadImg(byte[] data, string picurlDir)
{
String wlDir = picurlDir;//HttpContext.Current.Server.MapPath(picurlDir);
String wlDir2 = picurlDir + "/Thumb";
if (!Directory.Exists(wlDir))
{
Directory.CreateDirectory(wlDir);
}
if (!Directory.Exists(wlDir2))
{
Directory.CreateDirectory(wlDir2);
}
string filePath = DateTime.Now.ToString("yyyyMMddHHmmssfff");
filePath += new Random().Next(99).ToString("00");
try
{
MemoryStream ms = new MemoryStream(data);
Image image = System.Drawing.Image.FromStream(ms);
String wlfile = wlDir + "\\" + filePath + ".jpg";
image.Save(wlfile);
ms.Close();
return(wlfile);
}
catch (Exception ex)
{
LogServer.WriteLog(ex);
return("");
}
}
/// <summary>
/// GET 方式抓取网页信息
/// </summary>
/// <param name="url">地址</param>
/// <param name="encodingType">utf8、Default 网页编码格式</param>
/// <param name="ranAgent">随机的用户代理</param>
/// <returns>网页信息</returns>
public static string Gethtmlcode(string url, string encodingType = "utf-8", bool ranAgent = false)
{
if (string.IsNullOrEmpty(url) || (!url.Contains("http://") && !url.Contains("https://")))
{
return("");
}
int retry = 0;
while (true)
{
Encoding EncodingType;
try
{
HttpWebRequest mywr = (HttpWebRequest)WebRequest.Create(url);
mywr.Method = "GET";
mywr.Accept = "text/html, application/xhtml+xml, */*";
mywr.ContentType = "text/html";
mywr.UserAgent = ranAgent
? UserAgentList[Rang.Next(0, UserAgentList.Length)]
: "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/30.0.1599.101 Safari/537.36";
mywr.CookieContainer = null;
mywr.Headers.Add("Accept-Language", "zh-cn,en-us;");
mywr.Headers.Add("Accept-Encoding", "gzip, deflate");
mywr.KeepAlive = false;
mywr.Timeout = 60000;
if (mywr.CookieContainer == null)
{
mywr.CookieContainer = new CookieContainer();
}
HttpWebResponse mywrp = (HttpWebResponse)mywr.GetResponse();
Stream responseStream = mywrp.GetResponseStream();
if (mywrp.ContentEncoding.ToLower().Contains("gzip"))
{
if (responseStream != null)
{
responseStream = new GZipStream(responseStream, CompressionMode.Decompress);
}
}
int scode = (Int32)mywrp.StatusCode;
if (scode != 200)
{
return("Error StatusCode:" + scode);
}
// string strGethtml = "";
//if (responseStream == null) return strGethtml;
//StreamReader sr = new StreamReader(responseStream,
// encodingType == "utf8" ? Encoding.UTF8 : Encoding.Default);
//strGethtml = sr.ReadToEnd().Trim();
//mywrp.Close();
//sr.Close();
//return strGethtml;
string strGethtml = "";
string tempType = mywrp.Headers["Content-Type"] ?? "";
tempType = tempType.ToLower();
if (tempType.Contains("gbk") || tempType.Contains("gb2312") || tempType.Contains("utf-8"))
{
if (tempType.Contains("gbk") || tempType.Contains("gb2312"))
{
EncodingType = Encoding.GetEncoding("GBK");
}
else
{
EncodingType = Encoding.UTF8;
}
if (responseStream == null)
{
return(strGethtml);
}
StreamReader sr = new StreamReader(responseStream, EncodingType);
strGethtml = sr.ReadToEnd().Trim();
mywrp.Close();
sr.Close();
return(strGethtml);
}
if (mywrp.StatusCode == HttpStatusCode.Redirect)
{
strGethtml += string.Format("ResponseUri:{0}", mywrp.ResponseUri);
}
mywrp.Close();
return(strGethtml);
}
catch (Exception ex)
{
if (retry < 3)
{
retry++;
Thread.Sleep(1000);
continue;
}
LogServer.WriteLog("get url:" + url + ex, "HtmlAnalysis");
return("");
}
}
}
/// <summary>
/// POST方式抓取网页信息
/// </summary>
/// <param name="maiurl"></param>
/// <param name="paramurl"></param>
/// <param name="encode"></param>
/// <param name="ranAgent"></param>
/// <returns></returns>
public static string HttpRequestFromPost(string maiurl, string paramurl, string encode, bool ranAgent = true)
{
if (string.IsNullOrEmpty(maiurl) || (!maiurl.Contains("http://") && !maiurl.Contains("https://")))
{
return("");
}
int retry = 0;
while (true)
{
string strHtmlContent = "";
try
{
Encoding encoding = Encoding.GetEncoding(encode);
HttpWebRequest request = (HttpWebRequest)WebRequest.Create(maiurl);
request.Method = "POST";
request.UserAgent = ranAgent
? UserAgentList[Rang.Next(0, UserAgentList.Length)]
: "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/30.0.1599.101 Safari/537.36";
request.ContentType = "application/x-www-form-urlencoded";
request.AllowAutoRedirect = true;
byte[] postbyte = Encoding.UTF8.GetBytes(paramurl);
request.ContentLength = postbyte.Length;
Stream newStream = request.GetRequestStream();
newStream.Write(postbyte, 0, postbyte.Length); //把参数用流对象写入request对象中
newStream.Close();
HttpWebResponse response = (HttpWebResponse)request.GetResponse(); //获得服务器响应对象
Stream resStream = response.GetResponseStream(); //转成流对象
if (resStream != null)
{
string tempType = response.Headers["Content-Type"] ?? "";
tempType = tempType.ToLower();
if (tempType.Contains("gbk") || tempType.Contains("gb2312") || tempType.Contains("utf-8"))
{
if (tempType.Contains("gbk") || tempType.Contains("gb2312"))
{
encoding = Encoding.GetEncoding("GBK");
}
else
{
encoding = Encoding.UTF8;
}
}
StreamReader sr = new StreamReader(resStream, encoding);
strHtmlContent = sr.ReadToEnd();
}
response.Close();
return(strHtmlContent);
}
catch (Exception ex)
{
if (retry < 3)
{
retry++;
Thread.Sleep(1000);
continue;
}
LogServer.WriteLog("post url:" + maiurl + "?" + paramurl + ex, "HtmlAnalysis");
return("");
}
}
}
public string HttpRequest(string url, string param)
{
if (string.IsNullOrEmpty(url) || (!url.Contains("http://") && !url.Contains("https://")))
{
return("");
}
int retry = 0;
while (true)
{
string paraminfo = "";
string strGethtml = "";
try
{
if (!string.IsNullOrEmpty(param))
{
if (RequestMethod.ToUpper() == "POST")
{
paraminfo = param;
}
else
{
url = url + "?" + param;
}
}
HttpWebRequest mywr = (HttpWebRequest)WebRequest.Create(url);
mywr.Proxy = RequesProxy;
mywr.Method = RequestMethod;
mywr.Accept = RequestAccept;
mywr.ContentType = RequestContentType;
mywr.AllowAutoRedirect = RequestAutoRedirect;
if (!string.IsNullOrEmpty(RequestReferer))
{
mywr.Referer = RequestReferer;
}
mywr.UserAgent = RanAgent
? UserAgentList[Rang.Next(0, UserAgentList.Length)]
: RequestUserAgent;
if (Headers != null && Headers.Count > 0)
{
foreach (KeyValuePair <string, string> item in Headers)
{
mywr.Headers.Add(item.Key, item.Value);
}
}
mywr.KeepAlive = RequestKeepAlive;
mywr.Timeout = RequestTimeout;
if (RequestCookies.Count > 0)
{
mywr.CookieContainer = RequestCookies;
}
//把参数用流对象写入request对象中
if (paraminfo != "")
{
byte[] postbyte = Encoding.ASCII.GetBytes(paraminfo);
mywr.ContentLength = postbyte.Length;
Stream newStream = mywr.GetRequestStream();
newStream.Write(postbyte, 0, postbyte.Length);
newStream.Close();
}
HttpWebResponse mywrp = (HttpWebResponse)mywr.GetResponse();
//mywrp.Headers.ToString();
if (mywrp.ResponseUri.AbsoluteUri != url)
{
LogServer.WriteLog("url old:" + url + "new " + mywrp.ResponseUri, "UrlChange");
//return "";
}
if (ResultResponseHeader == null)
{
ResultResponseHeader = new Dictionary <string, string>();
}
else if (ResultResponseHeader.Count > 0)
{
ResultResponseHeader.Clear();
}
for (int i = 0; i < mywrp.Headers.Count; i++)
{
if (headKey.Contains(mywrp.Headers.Keys[i]))
{
continue;
}
ResultResponseHeader.Add(mywrp.Headers.Keys[i], mywrp.Headers.Get(i));
}
//SimulationCookie.SetCookie(mywr.Host, mywrp.Cookies);
Stream responseStream = mywrp.GetResponseStream();
if (mywrp.ContentEncoding.ToLower().Contains("gzip"))
{
if (responseStream != null)
{
responseStream = new GZipStream(responseStream, CompressionMode.Decompress);
}
}
int scode = (Int32)mywrp.StatusCode;
if (scode != 200)
{
return("Error StatusCode:" + scode);
}
Encoding encodingType;
string tempType = mywrp.Headers["Content-Type"] ?? "";
tempType = tempType.ToLower();
if (tempType.Contains("gbk") || tempType.Contains("gb2312") || tempType.Contains("utf-8"))
{
if (tempType.Contains("gbk") || tempType.Contains("gb2312"))
{
encodingType = Encoding.GetEncoding("GBK");
}
else
{
encodingType = Encoding.UTF8;
}
if (responseStream == null)
{
return(strGethtml);
}
StreamReader sr = new StreamReader(responseStream, encodingType);
strGethtml = sr.ReadToEnd().Trim();
mywrp.Close();
sr.Close();
return(strGethtml);
}
if (mywrp.StatusCode == HttpStatusCode.Redirect)
{
strGethtml += string.Format("ResponseUri:{0}", mywrp.ResponseUri);
}
mywrp.Close();
return(strGethtml);
//if (responseStream == null) return strGethtml;
//StreamReader sr = new StreamReader(responseStream, RequestEncoding);
//strGethtml = sr.ReadToEnd().Trim();
//mywrp.Close();
//sr.Close();
}
catch (Exception ex)
{
if (retry < 3)
{
retry++;
Thread.Sleep(1000);
continue;
}
if (ex is WebException)
{
var webEx = ex as WebException;
if (RequesProxy != null)
{
LogServer.WriteLog("url:" + url + "Proxy:" + RequesProxy.Address + "\t" + webEx.Message, "HtmlAnalysis");
}
else
{
LogServer.WriteLog("url:" + url + "" + webEx.Message, "HtmlAnalysis");
}
return("");
}
LogServer.WriteLog("get url:" + url + ex, "HtmlAnalysis");
return("");
}
}
}
