private void Init(FileFilter dFilter) {
filterList.Add(dFilter);
fileType = dFilter.Extension;
RegenFilter();
FilePath = null;
InitialDirectory = null;
DefaultFilename = null;
}
private void UpdateFilters ()
{
if (fileFilter == null)
fileFilter = new FileFilter ();
ArrayList filters = fileFilter.FilterArrayList;
fileTypeComboBox.BeginUpdate ();
fileTypeComboBox.Items.Clear ();
foreach (FilterStruct fs in filters) {
fileTypeComboBox.Items.Add (fs.filterName);
}
fileTypeComboBox.EndUpdate ();
mwfFileView.FilterArrayList = filters;
}
/// <summary>
/// 进行异步数据采集的方法，支持 Cookie，Proxy，Post，EventHandler，Url补全，Css补全，Css替换，正文提取
/// </summary>
/// <param name="url">所要进行采集的网站的有效 Url 地址</param>
/// <param name="method">提交数据所使用的方法</param>
/// <param name="QueryString">作为提交的参数集合</param>
/// <returns></returns>
public string Http(string url, Method method = Method.GET, string QueryString = "", FileFilter fileFilter = null)
{
_message = null;
_result = "";
_error = false;
_isUpdated = false;
Redo:
string filename = "";
string full_url = ((Method.GET == method) ? ((QueryString.Length > 0) ? ((url.IndexOf("?") > 0) ? (url + "&" + QueryString) : (url + "?" + QueryString)) : url) : url);
ServicePointManager.Expect100Continue = false;
string html = "";
HttpWebRequest http = (HttpWebRequest)WebRequest.Create(Uri.EscapeUriString(full_url));
// http.AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.None;// 是否执行压缩返回
if (_IsHaveCertification)
{
ServicePointManager.ServerCertificateValidationCallback = new System.Net.Security.RemoteCertificateValidationCallback(CheckValidationResult);
http = (HttpWebRequest)WebRequest.CreateDefault(new Uri(url));
http.Method = "GET";
}
#region autosave filename settings
if (_autosave)
{
if (_root == null)
{
_error = true;
_message = "You must set a directory to save the file when your autosave setting is true.";
return null;
}
Uri uri = http.RequestUri;
filename = _root.FullName + "\\" + uri.Host + uri.LocalPath.Replace("/", "\\");
string fileName_Temp = filename;
//string param = new Regex("(?<=(" + uri.LocalPath + "))[\\s\\S]*", RegexOptions.IgnoreCase).Match(uri.ToString()).Value.Trim();
//filename = _root.FullName + "\\" + uri.Host + uri.LocalPath.Replace("/", "\\");
//if (!String.IsNullOrEmpty(param))
//{
// param = param.Replace("/", "\\").Replace("?", "\\");
// filename += param;
//}
string query = new Regex(@"^\?", RegexOptions.IgnoreCase).Replace(uri.Query, "").Trim();
Dictionary<string, string> queryDict = new Dictionary<string, string>();
MatchCollection queryList = new Regex(@"(^|(?<=&))[^&=;]+=[\s\S]*?(?=($|&[^&=;]+=))", RegexOptions.IgnoreCase).Matches(query);
for (int i = 0; i < queryList.Count; i++)
{
queryDict.Add(new Regex(@"^[^&=;]+(?==)", RegexOptions.IgnoreCase).Match(queryList[i].Value).Value, new Regex(@"(?<==)[\s\S]*", RegexOptions.IgnoreCase).Match(queryList[i].Value).Value);
}
if (queryList.Count + PostData.Count > 0)
{
filename += "\\";
if (fileFilter != null)
{
for (int j = 0; j < fileFilter.Filter.Count; j++)
{
if ((Ajax.Method)((object[])fileFilter.Filter[j])[1] == Ajax.Method.POST)
{
if (PostData.ContainsKey((string)((object[])fileFilter.Filter[j])[0]))
{
filename += (new Regex(@"\\$", RegexOptions.IgnoreCase).IsMatch(filename) ? "" : "&") + (string)((object[])fileFilter.Filter[j])[0] + "=" + new Regex(@"[\\/:\*\?<>""\|]", RegexOptions.IgnoreCase).Replace(PostData[(string)((object[])fileFilter.Filter[j])[0]], new MatchEvaluator(encode));
}
}
else
{
if (queryDict.ContainsKey((string)((object[])fileFilter.Filter[j])[0]))
{
filename += (new Regex(@"\\$", RegexOptions.IgnoreCase).IsMatch(filename) ? "" : "&") + (string)((object[])fileFilter.Filter[j])[0] + "=" + new Regex(@"[\\/:\*\?<>""\|]", RegexOptions.IgnoreCase).Replace(queryDict[(string)((object[])fileFilter.Filter[j])[0]], new MatchEvaluator(encode));
}
}
}
}
else
{
filename += new Regex(@"[\\/:\*\?<>""\|]", RegexOptions.IgnoreCase).Replace(query, new MatchEvaluator(encode));
for (int j = 0; j < PostDataKeys.Length; j++)
{
filename += (new Regex(@"\\$", RegexOptions.IgnoreCase).IsMatch(filename) ? "" : "&") + PostDataKeys[j] + "=" + PostData[PostDataKeys[j]];
}
}
}
//if (query.Length > 0)
//{
// filename += "\\" + new Regex(@"[\\/:\*\?<>""\|]", RegexOptions.IgnoreCase).Replace(query, new MatchEvaluator(encode)); // System.Web.HttpUtility.UrlEncode(query);
//}
if (filename == fileName_Temp && !String.IsNullOrEmpty(query))
{
filename = Path.Combine(filename, query);
string root = Path.GetPathRoot(filename);
filename = Path.Combine(root, filename.Replace(root, "").Replace(":", "_").Replace("?", "_").Replace("<", "_").Replace(">", "_").Replace("|", "_").Trim());
}
filename += ".html";
//while (filename.Length >= 260)
//{
// filename = new Regex(@"(Chapter \d+)[\s\S]*?\\", RegexOptions.IgnoreCase).Replace(filename, "$1\\");
// filename = new Regex(@"(title \d+)[\s\S]*?\\", RegexOptions.IgnoreCase).Replace(filename, "$1\\");
// filename = filename.Substring(0, filename.LastIndexOf("\\") + 1) + Encryption.SHA1(filename.Substring(filename.LastIndexOf("\\") + 1, filename.Length - filename.LastIndexOf("\\") - 6)) + ".html";
//}
_filename = filename;
}
#endregion
_host = http.Host;
FileInfo fileinfo = null;
// filename = new Regex(@":", RegexOptions.IgnoreCase).Replace(filename, "_");E:\WorkSpace\BuildProcessTemplates\FastCase Tools\FastCase Tools\Program.cs
if (filename.Length > 0)
{
if (filename.Length >= 260)
{
filename = new Regex(@"(Chapter \d+)[\s\S]*?\\", RegexOptions.IgnoreCase).Replace(filename, "$1\\");
filename = new Regex(@"(title \d+)[\s\S]*?\\", RegexOptions.IgnoreCase).Replace(filename, "$1\\");
}
fileinfo = new FileInfo(filename);
}
if (fileinfo != null)
{
if (!_autoupdate && fileinfo.Exists)
{
Load(fileinfo);
_isUpdated = true;
return _result;
}
}
http.CookieContainer = new CookieContainer();
if (_cookies.Count > 0)
{
http.CookieContainer.Add(_cookies);
}
http.UserAgent = _Agent;
//http.Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8";
http.Accept = "*/*";
if (_Proxy != null)
{
http.Proxy = _Proxy;
}
#region post settings
if (method == Method.POST)
{
http.Method = "POST";
http.Accept = "application/json, text/javascript, */*";
http.KeepAlive = true;
byte[] pd = null;
switch (_charset.EncodingName)
{
case "Unicode (UTF-8)":
pd = new UTF8Encoding().GetBytes(QueryString);
break;
case "US-ASCII":
pd = new ASCIIEncoding().GetBytes(QueryString);
break;
case "Unicode":
pd = new UnicodeEncoding().GetBytes(QueryString);
break;
default:
pd = new UTF8Encoding().GetBytes(QueryString);
break;
}
http.ContentType = "application/x-www-form-urlencoded";
http.ContentLength = pd.Length;
Stream ps = http.GetRequestStream();
ps.Write(pd, 0, pd.Length);
ps.Close();
ps.Dispose();
}
#endregion
#region get responseStream
try
{
HttpWebResponse response = (HttpWebResponse)http.GetResponse();
//result = new HttpResult();
//#region base
////获取StatusCode
//result.StatusCode = response.StatusCode;
////获取StatusDescription
//result.StatusDescription = response.StatusDescription;
////获取Headers
//result.Header = response.Headers;
////获取最后访问的URl
//result.ResponseUri = response.ResponseUri.ToString();
////获取CookieCollection
//if (response.Cookies != null) result.CookieCollection = response.Cookies;
////获取set-cookie
//if (response.Headers["set-cookie"] != null) result.Cookie = response.Headers["set-cookie"];
//#endregion
foreach (Cookie cookie in response.Cookies)
{
_cookies.Add(cookie);
}
if (response.Headers.GetValues("Location") != null)
{
if (response.Headers.GetValues("Location")[0].Length > 0)
{
string jumpto = response.Headers.GetValues("Location")[0];
}
}
Stream s = response.GetResponseStream();
if (response.CharacterSet.Length > 0 && _autoEncoding)
{
if (String.IsNullOrEmpty(_ForceEncoding))
{
_charset = Encoding.GetEncoding(response.CharacterSet);
}
else
{
_charset = Encoding.GetEncoding(_ForceEncoding);
}
}
using (StreamReader sr = new StreamReader(s, _charset))
{
html = sr.ReadToEnd();
if (_AddFullPath)
{
html = UrlComplemented(response.ResponseUri, html);
}
//if (new Regex(@"<meta(?=[^<>]*?http-equiv=['""]?refresh)(?=[^<>]*?url=['""]?)[^<>]*?>", RegexOptions.IgnoreCase).IsMatch(html))
//{
// string dd = html;
// string jumpto = new Regex(@"(?<=<meta[^<>]*)(?<=url=['""]?)(?!['""]).*?(?=['""])", RegexOptions.IgnoreCase).Match(dd).Value;
// if (jumpto.EndsWith("&bhjs=0"))
// {
// jumpto = jumpto.Replace("&bhjs=0", "&bhjs=-1");
// }
// html = Http(jumpto);
//}
if (_AppendCss)
{
html = CssAppend(html);
}
_result = html;
sr.Close();
sr.Dispose();
}
s.Close();
s.Dispose();
response.Close();
}
catch (Exception ex)
{
if (ex.Message.Contains("The operation has timed out") || ex.Message == "The remote server returned an error: (504) Gateway Timeout")
{
goto Redo;
}
_error = true;
_message = ex.Message;
return null;
}
#endregion
if (_autosave)
{
if (_autoupdate || !fileinfo.Exists)
{
Save(fileinfo);
}
}
if (_event)
{
Completed(this, new EventArgs());
}
return html;
}