private void Init(FileFilter dFilter) { filterList.Add(dFilter); fileType = dFilter.Extension; RegenFilter(); FilePath = null; InitialDirectory = null; DefaultFilename = null; }
private void UpdateFilters () { if (fileFilter == null) fileFilter = new FileFilter (); ArrayList filters = fileFilter.FilterArrayList; fileTypeComboBox.BeginUpdate (); fileTypeComboBox.Items.Clear (); foreach (FilterStruct fs in filters) { fileTypeComboBox.Items.Add (fs.filterName); } fileTypeComboBox.EndUpdate (); mwfFileView.FilterArrayList = filters; }
/// <summary> /// 进行异步数据采集的方法,支持 Cookie,Proxy,Post,EventHandler,Url补全,Css补全,Css替换,正文提取 /// </summary> /// <param name="url">所要进行采集的网站的有效 Url 地址</param> /// <param name="method">提交数据所使用的方法</param> /// <param name="QueryString">作为提交的参数集合</param> /// <returns></returns> public string Http(string url, Method method = Method.GET, string QueryString = "", FileFilter fileFilter = null) { _message = null; _result = ""; _error = false; _isUpdated = false; Redo: string filename = ""; string full_url = ((Method.GET == method) ? ((QueryString.Length > 0) ? ((url.IndexOf("?") > 0) ? (url + "&" + QueryString) : (url + "?" + QueryString)) : url) : url); ServicePointManager.Expect100Continue = false; string html = ""; HttpWebRequest http = (HttpWebRequest)WebRequest.Create(Uri.EscapeUriString(full_url)); // http.AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.None;// 是否执行压缩返回 if (_IsHaveCertification) { ServicePointManager.ServerCertificateValidationCallback = new System.Net.Security.RemoteCertificateValidationCallback(CheckValidationResult); http = (HttpWebRequest)WebRequest.CreateDefault(new Uri(url)); http.Method = "GET"; } #region autosave filename settings if (_autosave) { if (_root == null) { _error = true; _message = "You must set a directory to save the file when your autosave setting is true."; return null; } Uri uri = http.RequestUri; filename = _root.FullName + "\\" + uri.Host + uri.LocalPath.Replace("/", "\\"); string fileName_Temp = filename; //string param = new Regex("(?<=(" + uri.LocalPath + "))[\\s\\S]*", RegexOptions.IgnoreCase).Match(uri.ToString()).Value.Trim(); //filename = _root.FullName + "\\" + uri.Host + uri.LocalPath.Replace("/", "\\"); //if (!String.IsNullOrEmpty(param)) //{ // param = param.Replace("/", "\\").Replace("?", "\\"); // filename += param; //} string query = new Regex(@"^\?", RegexOptions.IgnoreCase).Replace(uri.Query, "").Trim(); Dictionary<string, string> queryDict = new Dictionary<string, string>(); MatchCollection queryList = new Regex(@"(^|(?<=&))[^&=;]+=[\s\S]*?(?=($|&[^&=;]+=))", RegexOptions.IgnoreCase).Matches(query); for (int i = 0; i < queryList.Count; i++) { queryDict.Add(new Regex(@"^[^&=;]+(?==)", RegexOptions.IgnoreCase).Match(queryList[i].Value).Value, new Regex(@"(?<==)[\s\S]*", RegexOptions.IgnoreCase).Match(queryList[i].Value).Value); } if (queryList.Count + PostData.Count > 0) { filename += "\\"; if (fileFilter != null) { for (int j = 0; j < fileFilter.Filter.Count; j++) { if ((Ajax.Method)((object[])fileFilter.Filter[j])[1] == Ajax.Method.POST) { if (PostData.ContainsKey((string)((object[])fileFilter.Filter[j])[0])) { filename += (new Regex(@"\\$", RegexOptions.IgnoreCase).IsMatch(filename) ? "" : "&") + (string)((object[])fileFilter.Filter[j])[0] + "=" + new Regex(@"[\\/:\*\?<>""\|]", RegexOptions.IgnoreCase).Replace(PostData[(string)((object[])fileFilter.Filter[j])[0]], new MatchEvaluator(encode)); } } else { if (queryDict.ContainsKey((string)((object[])fileFilter.Filter[j])[0])) { filename += (new Regex(@"\\$", RegexOptions.IgnoreCase).IsMatch(filename) ? "" : "&") + (string)((object[])fileFilter.Filter[j])[0] + "=" + new Regex(@"[\\/:\*\?<>""\|]", RegexOptions.IgnoreCase).Replace(queryDict[(string)((object[])fileFilter.Filter[j])[0]], new MatchEvaluator(encode)); } } } } else { filename += new Regex(@"[\\/:\*\?<>""\|]", RegexOptions.IgnoreCase).Replace(query, new MatchEvaluator(encode)); for (int j = 0; j < PostDataKeys.Length; j++) { filename += (new Regex(@"\\$", RegexOptions.IgnoreCase).IsMatch(filename) ? "" : "&") + PostDataKeys[j] + "=" + PostData[PostDataKeys[j]]; } } } //if (query.Length > 0) //{ // filename += "\\" + new Regex(@"[\\/:\*\?<>""\|]", RegexOptions.IgnoreCase).Replace(query, new MatchEvaluator(encode)); // System.Web.HttpUtility.UrlEncode(query); //} if (filename == fileName_Temp && !String.IsNullOrEmpty(query)) { filename = Path.Combine(filename, query); string root = Path.GetPathRoot(filename); filename = Path.Combine(root, filename.Replace(root, "").Replace(":", "_").Replace("?", "_").Replace("<", "_").Replace(">", "_").Replace("|", "_").Trim()); } filename += ".html"; //while (filename.Length >= 260) //{ // filename = new Regex(@"(Chapter \d+)[\s\S]*?\\", RegexOptions.IgnoreCase).Replace(filename, "$1\\"); // filename = new Regex(@"(title \d+)[\s\S]*?\\", RegexOptions.IgnoreCase).Replace(filename, "$1\\"); // filename = filename.Substring(0, filename.LastIndexOf("\\") + 1) + Encryption.SHA1(filename.Substring(filename.LastIndexOf("\\") + 1, filename.Length - filename.LastIndexOf("\\") - 6)) + ".html"; //} _filename = filename; } #endregion _host = http.Host; FileInfo fileinfo = null; // filename = new Regex(@":", RegexOptions.IgnoreCase).Replace(filename, "_");E:\WorkSpace\BuildProcessTemplates\FastCase Tools\FastCase Tools\Program.cs if (filename.Length > 0) { if (filename.Length >= 260) { filename = new Regex(@"(Chapter \d+)[\s\S]*?\\", RegexOptions.IgnoreCase).Replace(filename, "$1\\"); filename = new Regex(@"(title \d+)[\s\S]*?\\", RegexOptions.IgnoreCase).Replace(filename, "$1\\"); } fileinfo = new FileInfo(filename); } if (fileinfo != null) { if (!_autoupdate && fileinfo.Exists) { Load(fileinfo); _isUpdated = true; return _result; } } http.CookieContainer = new CookieContainer(); if (_cookies.Count > 0) { http.CookieContainer.Add(_cookies); } http.UserAgent = _Agent; //http.Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"; http.Accept = "*/*"; if (_Proxy != null) { http.Proxy = _Proxy; } #region post settings if (method == Method.POST) { http.Method = "POST"; http.Accept = "application/json, text/javascript, */*"; http.KeepAlive = true; byte[] pd = null; switch (_charset.EncodingName) { case "Unicode (UTF-8)": pd = new UTF8Encoding().GetBytes(QueryString); break; case "US-ASCII": pd = new ASCIIEncoding().GetBytes(QueryString); break; case "Unicode": pd = new UnicodeEncoding().GetBytes(QueryString); break; default: pd = new UTF8Encoding().GetBytes(QueryString); break; } http.ContentType = "application/x-www-form-urlencoded"; http.ContentLength = pd.Length; Stream ps = http.GetRequestStream(); ps.Write(pd, 0, pd.Length); ps.Close(); ps.Dispose(); } #endregion #region get responseStream try { HttpWebResponse response = (HttpWebResponse)http.GetResponse(); //result = new HttpResult(); //#region base ////获取StatusCode //result.StatusCode = response.StatusCode; ////获取StatusDescription //result.StatusDescription = response.StatusDescription; ////获取Headers //result.Header = response.Headers; ////获取最后访问的URl //result.ResponseUri = response.ResponseUri.ToString(); ////获取CookieCollection //if (response.Cookies != null) result.CookieCollection = response.Cookies; ////获取set-cookie //if (response.Headers["set-cookie"] != null) result.Cookie = response.Headers["set-cookie"]; //#endregion foreach (Cookie cookie in response.Cookies) { _cookies.Add(cookie); } if (response.Headers.GetValues("Location") != null) { if (response.Headers.GetValues("Location")[0].Length > 0) { string jumpto = response.Headers.GetValues("Location")[0]; } } Stream s = response.GetResponseStream(); if (response.CharacterSet.Length > 0 && _autoEncoding) { if (String.IsNullOrEmpty(_ForceEncoding)) { _charset = Encoding.GetEncoding(response.CharacterSet); } else { _charset = Encoding.GetEncoding(_ForceEncoding); } } using (StreamReader sr = new StreamReader(s, _charset)) { html = sr.ReadToEnd(); if (_AddFullPath) { html = UrlComplemented(response.ResponseUri, html); } //if (new Regex(@"<meta(?=[^<>]*?http-equiv=['""]?refresh)(?=[^<>]*?url=['""]?)[^<>]*?>", RegexOptions.IgnoreCase).IsMatch(html)) //{ // string dd = html; // string jumpto = new Regex(@"(?<=<meta[^<>]*)(?<=url=['""]?)(?!['""]).*?(?=['""])", RegexOptions.IgnoreCase).Match(dd).Value; // if (jumpto.EndsWith("&bhjs=0")) // { // jumpto = jumpto.Replace("&bhjs=0", "&bhjs=-1"); // } // html = Http(jumpto); //} if (_AppendCss) { html = CssAppend(html); } _result = html; sr.Close(); sr.Dispose(); } s.Close(); s.Dispose(); response.Close(); } catch (Exception ex) { if (ex.Message.Contains("The operation has timed out") || ex.Message == "The remote server returned an error: (504) Gateway Timeout") { goto Redo; } _error = true; _message = ex.Message; return null; } #endregion if (_autosave) { if (_autoupdate || !fileinfo.Exists) { Save(fileinfo); } } if (_event) { Completed(this, new EventArgs()); } return html; }