public Bitmap getUrlImg(string SourceUrl, string UrlStr, ref CookieContainer CookieContainer, Configuration.NetConfigElement proxy)
{
Bitmap map_bitMap;
HttpWebRequest request = null;
Stream responseStream = null;
try
{
request = (HttpWebRequest)WebRequest.Create(UrlStr);
request.Timeout = 600000;
request.ReadWriteTimeout = 600000;
request.Referer = SourceUrl;
request.CookieContainer = CookieContainer;
if (proxy.proxy)
{
WebProxy webProxy = new WebProxy(); //定义一个网关对象
webProxy.Address = new Uri(proxy.proxyEntity.proxyAddess); //网关服务器:端口
webProxy.Credentials = new NetworkCredential(proxy.proxyEntity.proxyUid, proxy.proxyEntity.proxyPwd); //用戶名,密码
request.Proxy = webProxy;
}
responseStream = ((HttpWebResponse)request.GetResponse()).GetResponseStream();
Image original = Image.FromStream(responseStream);
map_bitMap = new Bitmap(original);
request.Abort();
responseStream.Close();
}
catch (Exception ex)
{
if (request != null)
{
request.Abort();
}
if (responseStream != null)
{
responseStream.Close();
}
throw ex;
}
return(map_bitMap);
}
public string getUrlInfByPost(string SourceUrl, string UrlStr, string postDataStr, string EnCode, ref CookieContainer CookieContainer, Configuration.NetConfigElement proxy)
{
string ContextStr = "";
HttpWebRequest request = null;
HttpWebResponse response = null;
Stream myRequestStream = null;
StreamWriter myStreamWriter = null;
Stream s = null;
MemoryStream ms = null;
MemoryStream js = null;
GZipStream g = null;
int c = 1024 * 10;
byte[] data = null;
try
{
ServicePointManager.Expect100Continue = false;
request = (HttpWebRequest)WebRequest.Create(UrlStr);
if (Program.CurrSpiderTimes == 1)
{
request.Timeout = Program.sysPara.BegTimeOut;
}
else
{
request.Timeout = Program.sysPara.BegTimeOut + Common.TypeConverter.ObjectToInt((Math.Pow(2, Program.CurrSpiderTimes - 1))) * Program.sysPara.IntervalTimeOut;
}
request.KeepAlive = false;
request.ServicePoint.Expect100Continue = false;
request.ServicePoint.UseNagleAlgorithm = false;
if (Program.CurrSpiderTimes == 1)
{
request.ReadWriteTimeout = Program.sysPara.BegRWTimeOut;
}
else
{
request.ReadWriteTimeout = Program.sysPara.BegRWTimeOut + Common.TypeConverter.ObjectToInt((Math.Pow(2, Program.CurrSpiderTimes - 1))) * Program.sysPara.IntervalRWTimeOut;
}
request.Method = "POST";
request.ContentType = "application/x-www-form-urlencoded";
request.Accept = "image/gif, image/x-xbitmap, image/jpeg, image/pjpeg, application/x-shockwave-flash, application/vnd.ms-excel, application/vnd.ms-powerpoint, application/msword, */*";
request.CookieContainer = CookieContainer;
request.UserAgent = "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1) ; .NET CLR 2.0.50727; .NET CLR 3.0.04506.648; .NET CLR 3.5.21022)";
request.Referer = SourceUrl;
HttpRequestCachePolicy policy = new HttpRequestCachePolicy(HttpRequestCacheLevel.NoCacheNoStore);
request.CachePolicy = policy;
if (proxy.proxy)
{
WebProxy webProxy = new WebProxy(); //定义一个网关对象
webProxy.Address = new Uri(proxy.proxyEntity.proxyAddess); //网关服务器:端口
webProxy.Credentials = new NetworkCredential(proxy.proxyEntity.proxyUid, proxy.proxyEntity.proxyPwd); //用戶名,密码
request.Proxy = webProxy;
}
ASCIIEncoding encoding = new ASCIIEncoding();
byte[] byte1 = encoding.GetBytes(postDataStr);
request.ContentLength = byte1.Length;
myRequestStream = request.GetRequestStream();
myRequestStream.Write(byte1, 0, byte1.Length);
myRequestStream.Close();
try
{
response = (HttpWebResponse)request.GetResponse();
}
catch (WebException)
{
response = (HttpWebResponse)request.GetResponse();
}
string ce = response.Headers[HttpResponseHeader.ContentEncoding];
response.Cookies = CookieContainer.GetCookies(request.RequestUri);
int ContentLength = (int)response.ContentLength;
s = response.GetResponseStream();
if (ContentLength < 0)
{
data = new byte[c];
ms = new MemoryStream();
int l = s.Read(data, 0, c);
while (l > 0)
{
ms.Write(data, 0, l);
l = s.Read(data, 0, c);
}
data = ms.ToArray();
ms.Close();
}
else
{
data = new byte[ContentLength];
int pos = 0;
int l;
while (ContentLength > 0)
{
l = s.Read(data, pos, ContentLength);
pos += l;
ContentLength -= l;
}
}
if (s != null)
{
s.Close();
}
if (request != null)
{
request.Abort();
}
string setCookieStr = response.GetResponseHeader("Set-Cookie");
if (response != null)
{
response.Close();
}
if (ce == "gzip")
{
js = new MemoryStream(); // 解压后的流
ms = new MemoryStream(data); // 用于解压的流
g = new GZipStream(ms, CompressionMode.Decompress);
byte[] buffer = new byte[c]; // 读数据缓冲区
int l = g.Read(buffer, 0, c); // 一次读 10K
while (l > 0)
{
js.Write(buffer, 0, l);
l = g.Read(buffer, 0, c);
}
g.Close();
ms.Close();
data = js.ToArray();
js.Close();
}
ContextStr = System.Text.Encoding.GetEncoding(EnCode).GetString(data);
}
catch (Exception ex)
{
if (request != null)
{
request.Abort();
}
if (response != null)
{
response.Close();
}
if (myRequestStream != null)
{
myRequestStream.Close();
}
if (myStreamWriter != null)
{
myStreamWriter.Close();
}
if (s != null)
{
s.Close();
}
if (ms != null)
{
ms.Close();
}
if (js != null)
{
js.Close();
}
if (g != null)
{
g.Close();
}
throw ex;
}
return(ContextStr);
}
public string GetUrlInfByPostRoom(string PostUrl, string SourceUrl, string UrlStr, string postDataStr, string EnCode, ref CookieContainer CookieContainer, Configuration.NetConfigElement proxy)
{
string ContextStr = "";
HttpWebRequest request = null;
HttpWebResponse response = null;
Stream myRequestStream = null;
StreamWriter myStreamWriter = null;
Stream s = null;
MemoryStream ms = null;
MemoryStream js = null;
GZipStream g = null;
try
{
ServicePointManager.Expect100Continue = false;
request = (HttpWebRequest)WebRequest.Create(UrlStr);
if (Program.CurrSpiderTimes == 1)
{
request.Timeout = Program.sysPara.BegTimeOut;
}
else
{
request.Timeout = Program.sysPara.BegTimeOut + Common.TypeConverter.ObjectToInt((Math.Pow(2, Program.CurrSpiderTimes - 1))) * Program.sysPara.IntervalTimeOut;
}
request.KeepAlive = false;
request.ServicePoint.Expect100Continue = false;
request.ServicePoint.UseNagleAlgorithm = false;
if (Program.CurrSpiderTimes == 1)
{
request.ReadWriteTimeout = Program.sysPara.BegRWTimeOut;
}
else
{
request.ReadWriteTimeout = Program.sysPara.BegRWTimeOut + Common.TypeConverter.ObjectToInt((Math.Pow(2, Program.CurrSpiderTimes - 1))) * Program.sysPara.IntervalRWTimeOut;
}
request.Method = "POST";
request.ContentType = "application/x-www-form-urlencoded";
request.Accept = "image/gif, image/x-xbitmap, image/jpeg, image/pjpeg, application/x-shockwave-flash, application/vnd.ms-excel, application/vnd.ms-powerpoint, application/msword, */*";
request.CookieContainer = CookieContainer;
request.UserAgent = "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1) ; .NET CLR 2.0.50727; .NET CLR 3.0.04506.648; .NET CLR 3.5.21022)";
request.Referer = SourceUrl;
if (proxy.proxy)
{
WebProxy webProxy = new WebProxy(); //定义一个网关对象
webProxy.Address = new Uri(proxy.proxyEntity.proxyAddess); //网关服务器:端口
webProxy.Credentials = new NetworkCredential(proxy.proxyEntity.proxyUid, proxy.proxyEntity.proxyPwd); //用戶名,密码
request.Proxy = webProxy;
}
ASCIIEncoding encoding = new ASCIIEncoding();
byte[] byte1 = encoding.GetBytes(postDataStr);
request.ContentLength = byte1.Length;
myRequestStream = request.GetRequestStream();
myRequestStream.Write(byte1, 0, byte1.Length);
myRequestStream.Close();
try
{
response = (HttpWebResponse)request.GetResponse();
}
catch (WebException)
{
response = (HttpWebResponse)request.GetResponse();
}
string ce = response.Headers[HttpResponseHeader.ContentEncoding];
response.Cookies = CookieContainer.GetCookies(request.RequestUri);
//PostUrl = "http://www.gzwsfdc.com/touming/RoomInfo.aspx";
ContextStr = GetUrlInfByGet(UrlStr, PostUrl, EnCode, ref CookieContainer, proxy);
}
catch (Exception ex)
{
if (request != null)
{
request.Abort();
}
if (response != null)
{
response.Close();
}
if (myRequestStream != null)
{
myRequestStream.Close();
}
if (myStreamWriter != null)
{
myStreamWriter.Close();
}
if (s != null)
{
s.Close();
}
if (ms != null)
{
ms.Close();
}
if (js != null)
{
js.Close();
}
if (g != null)
{
g.Close();
}
throw ex;
}
return(ContextStr);
}
public string GetUrlInfByGet1(string SourceUrl, string UrlStr, string EnCode, ref CookieContainer CookieContainer, Configuration.NetConfigElement proxy)
{
string ContextStr = "";
HttpWebRequest request = null;
HttpWebResponse response = null;
Stream s = null;
MemoryStream ms = null;
MemoryStream js = null;
GZipStream g = null;
int c = 1024 * 10;
byte[] data = null;
try
{
request = (HttpWebRequest)WebRequest.Create(UrlStr);
if (Program.CurrSpiderTimes == 1)
{
request.Timeout = Program.sysPara.BegTimeOut;
}
else
{
request.Timeout = Program.sysPara.BegTimeOut + Common.TypeConverter.ObjectToInt((Math.Pow(2, Program.CurrSpiderTimes - 1))) * Program.sysPara.IntervalTimeOut;
}
request.KeepAlive = false;
request.ServicePoint.Expect100Continue = false;
request.ServicePoint.UseNagleAlgorithm = false;
if (Program.CurrSpiderTimes == 1)
{
request.ReadWriteTimeout = Program.sysPara.BegRWTimeOut;
}
else
{
request.ReadWriteTimeout = Program.sysPara.BegRWTimeOut + Common.TypeConverter.ObjectToInt((Math.Pow(2, Program.CurrSpiderTimes - 1))) * Program.sysPara.IntervalRWTimeOut;
}
request.Method = "GET";
request.Accept = "text/html, application/xhtml+xml, */*";
request.CookieContainer = CookieContainer;
HttpRequestCachePolicy policy = new HttpRequestCachePolicy(HttpRequestCacheLevel.NoCacheNoStore);
request.CachePolicy = policy;
request.CookieContainer = new CookieContainer();
List <Cookie> clst = GetAllCookies(CookieContainer);
foreach (Cookie cc in clst)
{
if (cc.Name != "myComponents.myCKNEW")
{
//Regex regex = new Regex("(?<=DateTime=)(.|\n)*");
//Match m = regex.Match(cc.Value);
//string d = System.Web.HttpUtility.UrlEncode(DateTime.Now.AddSeconds(-1).ToString("yyyy/MM/dd HH:mm:ss"));
//cc.Value = cc.Value.Replace(m.Value, d);
request.CookieContainer.Add(cc);
}
}
request.UserAgent = "Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0) like Gecko";
request.Referer = SourceUrl;
request.Headers.Add("Accept-Language", "zh-CN");
request.Headers.Add("Accept-Encoding", "gzip, deflate");
SetHeaderValue(request.Headers, "Connection", "Keep-Alive");
//request.ContentType = "application/x-www-form-urlencoded";
//request.Headers.Add("Cache-Control", "no-cache");
//request.Connection = "Keep-Alive";
if (proxy.proxy)
{
WebProxy webProxy = new WebProxy(); //定义一个网关对象
webProxy.Address = new Uri(proxy.proxyEntity.proxyAddess); //网关服务器:端口
webProxy.Credentials = new NetworkCredential(proxy.proxyEntity.proxyUid, proxy.proxyEntity.proxyPwd); //用戶名,密码
request.Proxy = webProxy;
}
try
{
response = (HttpWebResponse)request.GetResponse();
}
catch (WebException)
{
response = (HttpWebResponse)request.GetResponse();
}
string ce = response.Headers[HttpResponseHeader.ContentEncoding];
response.Cookies = CookieContainer.GetCookies(request.RequestUri);
int ContentLength = (int)response.ContentLength;
s = response.GetResponseStream();
c = 1024 * 10;
if (ContentLength < 0)
{
data = new byte[c];
ms = new MemoryStream();
int l = s.Read(data, 0, c);
while (l > 0)
{
ms.Write(data, 0, l);
l = s.Read(data, 0, c);
}
data = ms.ToArray();
ms.Close();
}
else
{
data = new byte[ContentLength];
int pos = 0;
int l;
while (ContentLength > 0)
{
l = s.Read(data, pos, ContentLength);
pos += l;
ContentLength -= l;
}
}
if (s != null)
{
s.Close();
}
if (request != null)
{
request.Abort();
}
if (response != null)
{
response.Close();
}
if (ce == "gzip")
{
js = new MemoryStream(); // 解压后的流
ms = new MemoryStream(data); // 用于解压的流
g = new GZipStream(ms, CompressionMode.Decompress);
byte[] buffer = new byte[c]; // 读数据缓冲区
int l = g.Read(buffer, 0, c); // 一次读 10K
while (l > 0)
{
js.Write(buffer, 0, l);
l = g.Read(buffer, 0, c);
}
g.Close();
ms.Close();
data = js.ToArray();
js.Close();
}
ContextStr = System.Text.Encoding.GetEncoding(EnCode).GetString(data);
}
catch (Exception ex)
{
if (request != null)
{
request.Abort();
}
if (response != null)
{
response.Close();
}
if (s != null)
{
s.Close();
}
if (ms != null)
{
ms.Close();
}
if (js != null)
{
js.Close();
}
if (g != null)
{
g.Close();
}
throw ex;
}
return(ContextStr);
}
