/// <summary>
/// 防止获取HTML时出错
/// </summary>
/// <param name="webRequest"></param>
/// <returns></returns>
private static HttpWebResponse GetHttpWebResponse(HttpWebRequest webRequest)
{
try
{
return((HttpWebResponse)webRequest.GetResponse());
}
catch (WebException ex)
{
ErrorMessage.WriteLog("", "获取HTML代码出错,错误信息是：" + ex.Message + " 链接地址：" + ex.HelpLink);
return((HttpWebResponse)ex.Response);
}
}
/// <summary>
/// 作者：李延伟
/// 日期：2011-5-12
/// 功能：根据页面源代码区分编码格式；
/// </summary>
/// <param name="url"></param>
/// <returns></returns>
public static string GetHtmlNoFormat(string url)
{
try
{
if (String.IsNullOrEmpty(url))
{
return(string.Empty);
}
#region 统计开始
Stopwatch timer = new Stopwatch();
if (CrawTimeDebug)
{
timer.Start();
}
#endregion
HttpWebRequest httpWebRequest;
httpWebRequest = (HttpWebRequest)HttpWebRequest.Create(url);
httpWebRequest.Referer = url;
httpWebRequest.Method = "GET";
httpWebRequest.UserAgent = "User-Agent: Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.2; .NET CLR 1.0.3705;)";
HttpWebResponse httpWebResponse;
httpWebResponse = GetHttpWebResponse(httpWebRequest);
Stream responseStream = httpWebResponse.GetResponseStream();
StreamReader streamReader = new StreamReader(responseStream, Encoding.UTF8);
string html = streamReader.ReadToEnd();
streamReader.Close();
responseStream.Close();
#region 统计开始
if (CrawTimeDebug)
{
timer.Stop();
ErrorMessage.WriteLog("第一次获取HTML", "地址：" + url + ". 时间为: " + timer.Elapsed.TotalSeconds.ToString());
}
#endregion
return(html);
}
catch (Exception ex)
{
ErrorMessage.WriteLog("获取HTML代码出错", "地址：" + url + ". 错误信息为: " + ex.Message);
return(null);
}
}
/// <summary>
/// 李雅杰 20120817
/// 普通上传图片
/// </summary>
/// <param name="path">上传图片地址</param>
/// <returns></returns>
public bool Upload(string path)
{
if (!CheckInputData())
{
//return false;
}
//string imgSerPhyPath = m_RootPath + GetImagePath(m_ImageSn, "0x0");
//string sourceImagePhyPath = m_RootPath + GetImagePath(m_ImageSn, "origin");
try
{
// if (File.Exists(imgSerPhyPath)) File.Delete(imgSerPhyPath);
//if ((int)m_BuildImgUploadSite == 0)
// CheckDirectory(path, true);
CheckDirectory(path, true);
// 注释
if (this.m_FullImgFilePath != "" && this.m_FullImgFilePath != null)
{
File.Copy(this.m_FullImgFilePath, path, true);
}
else
{
this.m_PostedFile.SaveAs("1");
}
if ((int)m_BuildImgUploadSite != 0)
{
if (m_AddStampAry == null || m_ThumbnailAry == null)
{
LoadSizeFromConfig(m_BuildImgUploadSite);
}
return(BuildImage());
}
else
{
return(BuildImage());
}
}
catch (Exception ee)
{
ErrorMessage.WriteLog("", "错误信息为: " + ee.ToString());
m_ErrMsg = ee.Message;
return(false);
}
}
private void ImageTransfer(int w, int h, string opath, string npath)
{
try
{
Bitmap oldimg = new Bitmap(opath);
int nw = oldimg.Width;
int nh = oldimg.Height;
if (nw > w)
{
nw = w;
nh = (oldimg.Height * w) / oldimg.Width;
}
if (nh > h)
{
nh = h;
nw = (oldimg.Width * h) / oldimg.Height;
}
//Bitmap img = new Bitmap(oldimg, nw, nh);
Bitmap newimg = new Bitmap(w, h);
Graphics g = Graphics.FromImage(newimg);
g.SmoothingMode = SmoothingMode.HighQuality;
g.InterpolationMode = InterpolationMode.HighQualityBilinear;
g.Clear(Color.White);
//g.DrawImage(img, new Point((w - img.Width) / 2, (h - img.Height) / 2));
g.DrawImage(oldimg, (w - nw) / 2, (h - nh) / 2, nw, nh);
string dir = Path.GetDirectoryName(npath);
if (!Directory.Exists(dir))
{
Directory.CreateDirectory(dir);
}
ImageCodecInfo encoderInfo = GetEncoderInfoByExtension(this.m_ImageExtension);
EncoderParameter encoderParameter = new EncoderParameter(System.Drawing.Imaging.Encoder.Quality, 90L);
EncoderParameters encoderParameters = new EncoderParameters(1);
encoderParameters.Param[0] = encoderParameter;
newimg.Save(npath, encoderInfo, encoderParameters);
}
catch (Exception ex)
{
ErrorMessage.WriteLog("图片转换错误", "错误信息为: " + ex.ToString());
}
}
/// <summary>
/// 作者：李延伟
/// 日期：2011-5-12
/// 功能：根据meta获取网页的编码格式
/// </summary>
/// <param name="html"></param>
/// <returns></returns>
static Encoding GetEncoding(string html)
{
#region 统计开始
Stopwatch timer = new Stopwatch();
if (CrawTimeDebug)
{
timer.Start();
}
#endregion
string pattern = @"(?i)\bcharset=(?<charset>[-a-zA-Z_0-9]+)";
string charset = Regex.Match(html, pattern).Groups["charset"].Value;
#region 统计开始
if (CrawTimeDebug)
{
timer.Stop();
ErrorMessage.WriteLog("获取页面编码格式", " 正则时间: " + timer.Elapsed.TotalSeconds.ToString());
}
#endregion
try { return(Encoding.GetEncoding(charset)); }
catch (ArgumentException) { return(null); }
}
/// <summary>
/// 获取指定页面的HTML代码 用于国美抓报价
/// </summary>
/// <param name="postData">postData</param>
/// <param name="url">url</param>
/// <returns></returns>
public static string GetGomeHTML(string postData, string url)
{
#region 统计开始
Stopwatch timer = new Stopwatch();
if (CrawTimeDebug)
{
timer.Start();
}
#endregion
System.Net.ServicePointManager.Expect100Continue = false;
byte[] byteRequest = Encoding.UTF8.GetBytes(postData);
HttpWebRequest httpWebRequest;
httpWebRequest = (HttpWebRequest)HttpWebRequest.Create(url);
httpWebRequest.ContentType = "application/x-www-form-urlencoded";
httpWebRequest.Method = "Post";
httpWebRequest.KeepAlive = false;
httpWebRequest.ContentLength = byteRequest.Length;
httpWebRequest.Timeout = 99999999;
Stream stream = httpWebRequest.GetRequestStream();
stream.Write(byteRequest, 0, byteRequest.Length);
stream.Close();
HttpWebResponse httpWebResponse;
httpWebResponse = GetHttpWebResponse(httpWebRequest);
Stream responseStream = httpWebResponse.GetResponseStream();
StreamReader streamReader = new StreamReader(responseStream, Encoding.UTF8);
string html = streamReader.ReadToEnd();
streamReader.Close();
responseStream.Close();
#region 统计开始
if (CrawTimeDebug)
{
timer.Stop();
ErrorMessage.WriteLog("获取国美HTML", "地址：" + url + ". 时间为: " + timer.Elapsed.TotalSeconds.ToString());
}
#endregion
return(html);
}
/// <summary>
/// 作者：李延伟
/// 日期：2011-5-15
/// 功能：根据抓取规则获取价格
/// </summary>
/// <param name="ActionType">规则类型</param>
/// <param name="ActionData">抓取规则</param>
/// <param name="url">抓取地址</param>
/// <returns>报价</returns>
public static decimal GetPriceByModel(int ActionType, string ActionData, string url)
{
if (String.IsNullOrEmpty(url))
{
return(0);
}
//抓取的报价
decimal price = 0;
//抓取的HTML
string html = string.Empty;
//抓取的报价
string Result = string.Empty;
//提取商城地址
string MallUrl = GetMallUrl(url);
try
{
if (ActionType == 1)//正则抓取
{
html = GetHtml.GetHTML(MallUrl);
if (!String.IsNullOrEmpty(html))
{
Result = RunRegex(html, ActionData);
if (!string.IsNullOrEmpty(Result))
{
price = Convert.ToDecimal(Result);
}
else
{
price = 0;
}
}
}
else if (ActionType == 2) //特殊处理
{
object[] Param = new object[] { MallUrl };
price = Convert.ToDecimal(Common.ReflexST.StartMethod(ActionData, Param));
}
if (price == 0)
{
//抓不到报价
ErrorMessage.WriteLog("没有抓到报价", "抓取地址：" + url);
}
if (CrawTimeDebug)
{
ErrorMessage.WriteLog("抓取地址：" + url, "最终抓取的报价是：" + price);
}
return(price);
}
catch (Exception ex)
{
ErrorMessage.WriteLog("GetPriceByModel，错误信息为:" + ex.Message + "抓取规则是：" + ActionData, "抓取地址：" + MallUrl);
//抓报价出错 记录报错信息 ， URL
return(0);
}
}
public bool Upload(int uploadType)
{
if (!CheckInputData())
{
//return false;
}
string sizeConfigName = "";
switch (uploadType)
{
case 0: break;
case 1: this.RootPath = System.Configuration.ConfigurationManager.AppSettings["CategorySPImagesPath"].Trim(); //类目活动图片上传
sizeConfigName = "CategorySPImages";
break;
case 2: this.RootPath = System.Configuration.ConfigurationManager.AppSettings["EBWebsiteImagesPath"].Trim(); //电商图片上传
sizeConfigName = "EBWebsiteImages";
break;
case 3: this.RootPath = System.Configuration.ConfigurationManager.AppSettings["ProductSummaryImagesPath"].Trim(); //电商图片上传
sizeConfigName = "ProductSummaryImages";
break;
case 4: this.RootPath = System.Configuration.ConfigurationManager.AppSettings["SeoTitleImagePath"].Trim(); //手工内容图片上传
sizeConfigName = "SeoTitleImages";
break;
case 5: this.RootPath = System.Configuration.ConfigurationManager.AppSettings["QHMImage"].Trim() + "\\QHMProductImage"; //抢红米产品图片上传
sizeConfigName = "QHMProductImages";
break;
case 6: this.RootPath = System.Configuration.ConfigurationManager.AppSettings["QHMImage"].Trim() + "\\QHMHome"; //抢红米首页图片上传
sizeConfigName = "QHMHomeImages";
break;
case 7: this.RootPath = System.Configuration.ConfigurationManager.AppSettings["QHMImage"].Trim() + "\\QHMContentImage"; //抢红米产品描述图片上传
sizeConfigName = "QHMContentImages";
break;
case 8: this.RootPath = System.Configuration.ConfigurationManager.AppSettings["QFQImage"].Trim() + "\\QFQHome"; //抢红米产品描述图片上传
sizeConfigName = "QFQImages";
break;
case 9: this.RootPath = System.Configuration.ConfigurationManager.AppSettings["QHMImage"].Trim() + "\\TwoClassImages"; //专题二级分类图片上传
sizeConfigName = "TwoClassImages";
break;
case 10: this.RootPath = System.Configuration.ConfigurationManager.AppSettings["QHMImage"].Trim() + "\\LotteryImage\\"; //牛败商城转盘抽奖图片上传
sizeConfigName = "LotteryImages";
break;
case 11: this.RootPath = System.Configuration.ConfigurationManager.AppSettings["QHMImage"].Trim() + "\\WX0BuyImages\\"; //微信0元购推广图片上传
sizeConfigName = "QHMProductImages"; //使用抢红米产品图片尺寸
break;
case 12: this.RootPath = System.Configuration.ConfigurationManager.AppSettings["QHMImage"].Trim() + "\\QHMTopicImages\\"; //牛败商城专题图
sizeConfigName = "QHMTopicImages";
break;
case 13: this.RootPath = System.Configuration.ConfigurationManager.AppSettings["QHMImage"].Trim() + "\\QHMBrandImages\\"; //牛败商城品牌图
sizeConfigName = "QHMBrandImages";
break;
}
string imgSerPhyPath = m_RootPath + GetImagePath(m_ImageSn, "0x0");
string sourceImagePhyPath = m_RootPath + GetImagePath(m_ImageSn, "origin");
ErrorMessage.WriteLog("", "imgSerPhyPath:" + imgSerPhyPath + " sourceImagePhyPath:" + sourceImagePhyPath);
try
{
// if (File.Exists(imgSerPhyPath)) File.Delete(imgSerPhyPath);
if ((int)m_BuildImgUploadSite == 0)
{
CheckDirectory(imgSerPhyPath, true);
}
CheckDirectory(sourceImagePhyPath, true);
// 注释
if (this.m_FullImgFilePath != "" && this.m_FullImgFilePath != null)
{
File.Copy(this.m_FullImgFilePath, sourceImagePhyPath, true);
}
else
{
this.m_PostedFile.SaveAs(sourceImagePhyPath);
}
if ((int)m_BuildImgUploadSite != 0)
{
if (m_BuildImgUploadSite == ImgUploadSite.Both)
{
bool fromConfig = (m_AddStampAry == null || m_ThumbnailAry == null);
bool success = false;
m_BuildImgUploadSite = ImgUploadSite.IT168;
if (fromConfig)
{
LoadSizeFromConfig(ImgUploadSite.IT168, sizeConfigName);
}
success = BuildImage();
m_BuildImgUploadSite = ImgUploadSite.PCPOP;
if (fromConfig)
{
LoadSizeFromConfig(ImgUploadSite.PCPOP, sizeConfigName);
}
return(BuildImage() && success);
}
else
{
if (m_AddStampAry == null || m_ThumbnailAry == null)
{
LoadSizeFromConfig(m_BuildImgUploadSite, sizeConfigName);
}
return(BuildImage());
}
}
else
{
return(BuildImage());
}
}
catch (Exception ee)
{
ErrorMessage.WriteLog("", "错误信息为: " + ee.ToString());
m_ErrMsg = ee.Message;
return(false);
}
}
public bool Upload()
{
if (!CheckInputData())
{
//return false;
}
string imgSerPhyPath = m_RootPath + GetImagePath(m_ImageSn, "0x0");
string sourceImagePhyPath = m_RootPath + GetImagePath(m_ImageSn, "origin");
try
{
// if (File.Exists(imgSerPhyPath)) File.Delete(imgSerPhyPath);
if ((int)m_BuildImgUploadSite == 0)
{
CheckDirectory(imgSerPhyPath, true);
}
CheckDirectory(sourceImagePhyPath, true);
// 注释
if (this.m_FullImgFilePath != "" && this.m_FullImgFilePath != null)
{
File.Copy(this.m_FullImgFilePath, sourceImagePhyPath, true);
}
else
{
this.m_PostedFile.SaveAs(sourceImagePhyPath);
}
if ((int)m_BuildImgUploadSite != 0)
{
if (m_BuildImgUploadSite == ImgUploadSite.Both)
{
bool fromConfig = (m_AddStampAry == null || m_ThumbnailAry == null);
bool success = false;
m_BuildImgUploadSite = ImgUploadSite.IT168;
if (fromConfig)
{
LoadSizeFromConfig(ImgUploadSite.IT168);
}
success = BuildImage();
m_BuildImgUploadSite = ImgUploadSite.PCPOP;
if (fromConfig)
{
LoadSizeFromConfig(ImgUploadSite.PCPOP);
}
return(BuildImage() && success);
}
else
{
if (m_AddStampAry == null || m_ThumbnailAry == null)
{
LoadSizeFromConfig(m_BuildImgUploadSite);
}
return(BuildImage());
}
}
else
{
return(BuildImage());
}
}
catch (Exception ee)
{
ErrorMessage.WriteLog("", "错误信息为: " + ee.ToString());
m_ErrMsg = ee.Message;
return(false);
}
}
