/// <summary> /// 防止获取HTML时出错 /// </summary> /// <param name="webRequest"></param> /// <returns></returns> private static HttpWebResponse GetHttpWebResponse(HttpWebRequest webRequest) { try { return((HttpWebResponse)webRequest.GetResponse()); } catch (WebException ex) { ErrorMessage.WriteLog("", "获取HTML代码出错,错误信息是:" + ex.Message + " 链接地址:" + ex.HelpLink); return((HttpWebResponse)ex.Response); } }
/// <summary> /// 作者:李延伟 /// 日期:2011-5-12 /// 功能:根据页面源代码区分编码格式; /// </summary> /// <param name="url"></param> /// <returns></returns> public static string GetHtmlNoFormat(string url) { try { if (String.IsNullOrEmpty(url)) { return(string.Empty); } #region 统计开始 Stopwatch timer = new Stopwatch(); if (CrawTimeDebug) { timer.Start(); } #endregion HttpWebRequest httpWebRequest; httpWebRequest = (HttpWebRequest)HttpWebRequest.Create(url); httpWebRequest.Referer = url; httpWebRequest.Method = "GET"; httpWebRequest.UserAgent = "User-Agent: Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.2; .NET CLR 1.0.3705;)"; HttpWebResponse httpWebResponse; httpWebResponse = GetHttpWebResponse(httpWebRequest); Stream responseStream = httpWebResponse.GetResponseStream(); StreamReader streamReader = new StreamReader(responseStream, Encoding.UTF8); string html = streamReader.ReadToEnd(); streamReader.Close(); responseStream.Close(); #region 统计开始 if (CrawTimeDebug) { timer.Stop(); ErrorMessage.WriteLog("第一次获取HTML", "地址:" + url + ". 时间为: " + timer.Elapsed.TotalSeconds.ToString()); } #endregion return(html); } catch (Exception ex) { ErrorMessage.WriteLog("获取HTML代码出错", "地址:" + url + ". 错误信息为: " + ex.Message); return(null); } }
/// <summary> /// 李雅杰 20120817 /// 普通上传图片 /// </summary> /// <param name="path">上传图片地址</param> /// <returns></returns> public bool Upload(string path) { if (!CheckInputData()) { //return false; } //string imgSerPhyPath = m_RootPath + GetImagePath(m_ImageSn, "0x0"); //string sourceImagePhyPath = m_RootPath + GetImagePath(m_ImageSn, "origin"); try { // if (File.Exists(imgSerPhyPath)) File.Delete(imgSerPhyPath); //if ((int)m_BuildImgUploadSite == 0) // CheckDirectory(path, true); CheckDirectory(path, true); // 注释 if (this.m_FullImgFilePath != "" && this.m_FullImgFilePath != null) { File.Copy(this.m_FullImgFilePath, path, true); } else { this.m_PostedFile.SaveAs("1"); } if ((int)m_BuildImgUploadSite != 0) { if (m_AddStampAry == null || m_ThumbnailAry == null) { LoadSizeFromConfig(m_BuildImgUploadSite); } return(BuildImage()); } else { return(BuildImage()); } } catch (Exception ee) { ErrorMessage.WriteLog("", "错误信息为: " + ee.ToString()); m_ErrMsg = ee.Message; return(false); } }
private void ImageTransfer(int w, int h, string opath, string npath) { try { Bitmap oldimg = new Bitmap(opath); int nw = oldimg.Width; int nh = oldimg.Height; if (nw > w) { nw = w; nh = (oldimg.Height * w) / oldimg.Width; } if (nh > h) { nh = h; nw = (oldimg.Width * h) / oldimg.Height; } //Bitmap img = new Bitmap(oldimg, nw, nh); Bitmap newimg = new Bitmap(w, h); Graphics g = Graphics.FromImage(newimg); g.SmoothingMode = SmoothingMode.HighQuality; g.InterpolationMode = InterpolationMode.HighQualityBilinear; g.Clear(Color.White); //g.DrawImage(img, new Point((w - img.Width) / 2, (h - img.Height) / 2)); g.DrawImage(oldimg, (w - nw) / 2, (h - nh) / 2, nw, nh); string dir = Path.GetDirectoryName(npath); if (!Directory.Exists(dir)) { Directory.CreateDirectory(dir); } ImageCodecInfo encoderInfo = GetEncoderInfoByExtension(this.m_ImageExtension); EncoderParameter encoderParameter = new EncoderParameter(System.Drawing.Imaging.Encoder.Quality, 90L); EncoderParameters encoderParameters = new EncoderParameters(1); encoderParameters.Param[0] = encoderParameter; newimg.Save(npath, encoderInfo, encoderParameters); } catch (Exception ex) { ErrorMessage.WriteLog("图片转换错误", "错误信息为: " + ex.ToString()); } }
/// <summary> /// 作者:李延伟 /// 日期:2011-5-12 /// 功能:根据meta获取网页的编码格式 /// </summary> /// <param name="html"></param> /// <returns></returns> static Encoding GetEncoding(string html) { #region 统计开始 Stopwatch timer = new Stopwatch(); if (CrawTimeDebug) { timer.Start(); } #endregion string pattern = @"(?i)\bcharset=(?<charset>[-a-zA-Z_0-9]+)"; string charset = Regex.Match(html, pattern).Groups["charset"].Value; #region 统计开始 if (CrawTimeDebug) { timer.Stop(); ErrorMessage.WriteLog("获取页面编码格式", " 正则时间: " + timer.Elapsed.TotalSeconds.ToString()); } #endregion try { return(Encoding.GetEncoding(charset)); } catch (ArgumentException) { return(null); } }
/// <summary> /// 获取指定页面的HTML代码 用于国美抓报价 /// </summary> /// <param name="postData">postData</param> /// <param name="url">url</param> /// <returns></returns> public static string GetGomeHTML(string postData, string url) { #region 统计开始 Stopwatch timer = new Stopwatch(); if (CrawTimeDebug) { timer.Start(); } #endregion System.Net.ServicePointManager.Expect100Continue = false; byte[] byteRequest = Encoding.UTF8.GetBytes(postData); HttpWebRequest httpWebRequest; httpWebRequest = (HttpWebRequest)HttpWebRequest.Create(url); httpWebRequest.ContentType = "application/x-www-form-urlencoded"; httpWebRequest.Method = "Post"; httpWebRequest.KeepAlive = false; httpWebRequest.ContentLength = byteRequest.Length; httpWebRequest.Timeout = 99999999; Stream stream = httpWebRequest.GetRequestStream(); stream.Write(byteRequest, 0, byteRequest.Length); stream.Close(); HttpWebResponse httpWebResponse; httpWebResponse = GetHttpWebResponse(httpWebRequest); Stream responseStream = httpWebResponse.GetResponseStream(); StreamReader streamReader = new StreamReader(responseStream, Encoding.UTF8); string html = streamReader.ReadToEnd(); streamReader.Close(); responseStream.Close(); #region 统计开始 if (CrawTimeDebug) { timer.Stop(); ErrorMessage.WriteLog("获取国美HTML", "地址:" + url + ". 时间为: " + timer.Elapsed.TotalSeconds.ToString()); } #endregion return(html); }
/// <summary> /// 作者:李延伟 /// 日期:2011-5-15 /// 功能:根据抓取规则获取价格 /// </summary> /// <param name="ActionType">规则类型</param> /// <param name="ActionData">抓取规则</param> /// <param name="url">抓取地址</param> /// <returns>报价</returns> public static decimal GetPriceByModel(int ActionType, string ActionData, string url) { if (String.IsNullOrEmpty(url)) { return(0); } //抓取的报价 decimal price = 0; //抓取的HTML string html = string.Empty; //抓取的报价 string Result = string.Empty; //提取商城地址 string MallUrl = GetMallUrl(url); try { if (ActionType == 1)//正则抓取 { html = GetHtml.GetHTML(MallUrl); if (!String.IsNullOrEmpty(html)) { Result = RunRegex(html, ActionData); if (!string.IsNullOrEmpty(Result)) { price = Convert.ToDecimal(Result); } else { price = 0; } } } else if (ActionType == 2) //特殊处理 { object[] Param = new object[] { MallUrl }; price = Convert.ToDecimal(Common.ReflexST.StartMethod(ActionData, Param)); } if (price == 0) { //抓不到报价 ErrorMessage.WriteLog("没有抓到报价", "抓取地址:" + url); } if (CrawTimeDebug) { ErrorMessage.WriteLog("抓取地址:" + url, "最终抓取的报价是:" + price); } return(price); } catch (Exception ex) { ErrorMessage.WriteLog("GetPriceByModel,错误信息为:" + ex.Message + "抓取规则是:" + ActionData, "抓取地址:" + MallUrl); //抓报价出错 记录报错信息 , URL return(0); } }
public bool Upload(int uploadType) { if (!CheckInputData()) { //return false; } string sizeConfigName = ""; switch (uploadType) { case 0: break; case 1: this.RootPath = System.Configuration.ConfigurationManager.AppSettings["CategorySPImagesPath"].Trim(); //类目活动图片上传 sizeConfigName = "CategorySPImages"; break; case 2: this.RootPath = System.Configuration.ConfigurationManager.AppSettings["EBWebsiteImagesPath"].Trim(); //电商图片上传 sizeConfigName = "EBWebsiteImages"; break; case 3: this.RootPath = System.Configuration.ConfigurationManager.AppSettings["ProductSummaryImagesPath"].Trim(); //电商图片上传 sizeConfigName = "ProductSummaryImages"; break; case 4: this.RootPath = System.Configuration.ConfigurationManager.AppSettings["SeoTitleImagePath"].Trim(); //手工内容图片上传 sizeConfigName = "SeoTitleImages"; break; case 5: this.RootPath = System.Configuration.ConfigurationManager.AppSettings["QHMImage"].Trim() + "\\QHMProductImage"; //抢红米产品图片上传 sizeConfigName = "QHMProductImages"; break; case 6: this.RootPath = System.Configuration.ConfigurationManager.AppSettings["QHMImage"].Trim() + "\\QHMHome"; //抢红米首页图片上传 sizeConfigName = "QHMHomeImages"; break; case 7: this.RootPath = System.Configuration.ConfigurationManager.AppSettings["QHMImage"].Trim() + "\\QHMContentImage"; //抢红米产品描述图片上传 sizeConfigName = "QHMContentImages"; break; case 8: this.RootPath = System.Configuration.ConfigurationManager.AppSettings["QFQImage"].Trim() + "\\QFQHome"; //抢红米产品描述图片上传 sizeConfigName = "QFQImages"; break; case 9: this.RootPath = System.Configuration.ConfigurationManager.AppSettings["QHMImage"].Trim() + "\\TwoClassImages"; //专题二级分类图片上传 sizeConfigName = "TwoClassImages"; break; case 10: this.RootPath = System.Configuration.ConfigurationManager.AppSettings["QHMImage"].Trim() + "\\LotteryImage\\"; //牛败商城转盘抽奖图片上传 sizeConfigName = "LotteryImages"; break; case 11: this.RootPath = System.Configuration.ConfigurationManager.AppSettings["QHMImage"].Trim() + "\\WX0BuyImages\\"; //微信0元购推广图片上传 sizeConfigName = "QHMProductImages"; //使用抢红米产品图片尺寸 break; case 12: this.RootPath = System.Configuration.ConfigurationManager.AppSettings["QHMImage"].Trim() + "\\QHMTopicImages\\"; //牛败商城专题图 sizeConfigName = "QHMTopicImages"; break; case 13: this.RootPath = System.Configuration.ConfigurationManager.AppSettings["QHMImage"].Trim() + "\\QHMBrandImages\\"; //牛败商城品牌图 sizeConfigName = "QHMBrandImages"; break; } string imgSerPhyPath = m_RootPath + GetImagePath(m_ImageSn, "0x0"); string sourceImagePhyPath = m_RootPath + GetImagePath(m_ImageSn, "origin"); ErrorMessage.WriteLog("", "imgSerPhyPath:" + imgSerPhyPath + " sourceImagePhyPath:" + sourceImagePhyPath); try { // if (File.Exists(imgSerPhyPath)) File.Delete(imgSerPhyPath); if ((int)m_BuildImgUploadSite == 0) { CheckDirectory(imgSerPhyPath, true); } CheckDirectory(sourceImagePhyPath, true); // 注释 if (this.m_FullImgFilePath != "" && this.m_FullImgFilePath != null) { File.Copy(this.m_FullImgFilePath, sourceImagePhyPath, true); } else { this.m_PostedFile.SaveAs(sourceImagePhyPath); } if ((int)m_BuildImgUploadSite != 0) { if (m_BuildImgUploadSite == ImgUploadSite.Both) { bool fromConfig = (m_AddStampAry == null || m_ThumbnailAry == null); bool success = false; m_BuildImgUploadSite = ImgUploadSite.IT168; if (fromConfig) { LoadSizeFromConfig(ImgUploadSite.IT168, sizeConfigName); } success = BuildImage(); m_BuildImgUploadSite = ImgUploadSite.PCPOP; if (fromConfig) { LoadSizeFromConfig(ImgUploadSite.PCPOP, sizeConfigName); } return(BuildImage() && success); } else { if (m_AddStampAry == null || m_ThumbnailAry == null) { LoadSizeFromConfig(m_BuildImgUploadSite, sizeConfigName); } return(BuildImage()); } } else { return(BuildImage()); } } catch (Exception ee) { ErrorMessage.WriteLog("", "错误信息为: " + ee.ToString()); m_ErrMsg = ee.Message; return(false); } }
public bool Upload() { if (!CheckInputData()) { //return false; } string imgSerPhyPath = m_RootPath + GetImagePath(m_ImageSn, "0x0"); string sourceImagePhyPath = m_RootPath + GetImagePath(m_ImageSn, "origin"); try { // if (File.Exists(imgSerPhyPath)) File.Delete(imgSerPhyPath); if ((int)m_BuildImgUploadSite == 0) { CheckDirectory(imgSerPhyPath, true); } CheckDirectory(sourceImagePhyPath, true); // 注释 if (this.m_FullImgFilePath != "" && this.m_FullImgFilePath != null) { File.Copy(this.m_FullImgFilePath, sourceImagePhyPath, true); } else { this.m_PostedFile.SaveAs(sourceImagePhyPath); } if ((int)m_BuildImgUploadSite != 0) { if (m_BuildImgUploadSite == ImgUploadSite.Both) { bool fromConfig = (m_AddStampAry == null || m_ThumbnailAry == null); bool success = false; m_BuildImgUploadSite = ImgUploadSite.IT168; if (fromConfig) { LoadSizeFromConfig(ImgUploadSite.IT168); } success = BuildImage(); m_BuildImgUploadSite = ImgUploadSite.PCPOP; if (fromConfig) { LoadSizeFromConfig(ImgUploadSite.PCPOP); } return(BuildImage() && success); } else { if (m_AddStampAry == null || m_ThumbnailAry == null) { LoadSizeFromConfig(m_BuildImgUploadSite); } return(BuildImage()); } } else { return(BuildImage()); } } catch (Exception ee) { ErrorMessage.WriteLog("", "错误信息为: " + ee.ToString()); m_ErrMsg = ee.Message; return(false); } }