/// <summary> /// 为请求准备参数 /// </summary> /// <param name="item">参数列表</param> /// <param name="_Encoding">读取数据时的编码方式</param> private HttpWebRequest SetRequest(HttpItem item, string desturl = null, string post = null) { var url = desturl ?? item.URL; if (url == null) { return(null); } if (url.Contains("http") == false) { url = "http://" + url; } // 验证证书 if (url.Contains("https")) { ServicePointManager.ServerCertificateValidationCallback = (sender, certificate, chain, sslPolicyErrors) => true; } //初始化对像,并设置请求的URL地址 var request = (HttpWebRequest)WebRequest.Create(GetUrl(url)); SetRequest(item, request, desturl, post); encoding = AttributeHelper.GetEncoding(item.Encoding); return(request); }
public override IEnumerable <IFreeDocument> Execute(IEnumerable <IFreeDocument> documents) { foreach (var document in documents) { string item = document[Column].ToString(); var mypath = document.Query(Path); var path = document.Query(mypath); DirectoryInfo directoryInfo = new DirectoryInfo(path); var folder = directoryInfo.Parent; if (folder == null) { continue; } if (!folder.Exists) { folder.Create(); } var url = document[Column].ToString(); if (string.IsNullOrEmpty(url)) { continue; } File.WriteAllText(mypath, item, AttributeHelper.GetEncoding(EncodingType)); yield return(document); } }
public override IEnumerable <IFreeDocument> Execute(IEnumerable <IFreeDocument> documents) { foreach (var document in documents) { string item = document[Column].ToString(); var mypath = document.Query(Path); File.WriteAllText(mypath, item, AttributeHelper.GetEncoding(EncodingType)); yield return(document); } }
public override object TransformData(IFreeDocument datas) { string item = datas[Column].ToString(); var res = buffHelper.Get(item); if (res != null) { return(res); } else { var content = File.ReadAllText(item, AttributeHelper.GetEncoding(EncodingType)); buffHelper.Set(item, content); return(content); } }
/// <summary> /// 为请求准备参数 /// </summary> /// <param name="item">参数列表</param> /// <param name="_Encoding">读取数据时的编码方式</param> private HttpWebRequest SetRequest(HttpItem item, string desturl = null, string post = null) { var url = desturl ?? item.URL; if (url == null) { return(null); } if (url.Contains("http") == false) { url = "http://" + url; } // 验证证书 if (url.Contains("https")) { ServicePointManager.ServerCertificateValidationCallback = (sender, certificate, chain, sslPolicyErrors) => true; } //初始化对像,并设置请求的URL地址 var request = (HttpWebRequest)WebRequest.Create(GetUrl(url)); var docu = item.GetHeaderParameter(); // 设置代理 //SetProxy(item); //请求方式Get或者Post request.Method = item.Method.ToString(); request.Timeout = item.Timeout; request.ReadWriteTimeout = item.ReadWriteTimeout; //Accept request.Headers = new WebHeaderCollection(); if (docu["Headers"].ToString() != "") { var str = docu["Headers"].ToString().Split('\n'); foreach (var s in str) { var ms = s.Split(':'); if (ms.Length != 2) { continue; } var key = ms[0].Trim(); var value = ms[1].Trim(); if (SetHeaderValue(request.Headers, key, value) == false) { request.Headers.Add(key, value); } } } request.Accept = docu["Accept"].ToString(); //ContentType返回类型 request.ContentType = docu["Content_Type"].ToString(); //UserAgent客户端的访问类型,包括浏览器版本和操作系统信息 request.UserAgent = docu["User-Agent"].ToString(); var host = docu["Host"].ToString(); //if (string.IsNullOrEmpty(host) == false) // request.Host = host; encoding = AttributeHelper.GetEncoding(item.Encoding); //设置Cookie var cookie = docu["Cookie"].ToString(); if (!string.IsNullOrEmpty(cookie)) { request.Headers[HttpRequestHeader.Cookie] = cookie; } //来源地址 request.Referer = docu["Referer"].ToString(); //是否执行跳转功能 request.AllowAutoRedirect = item.Allowautoredirect; //设置Post数据 string postdata = null; if (post == null) { postdata = item.Postdata; } else { postdata = post; } //验证在得到结果时是否有传入数据 if (!string.IsNullOrEmpty(postdata) && request.Method.Trim().ToLower().Contains("post")) { var buffer = Encoding.Default.GetBytes(postdata); request.ContentLength = buffer.Length; request.GetRequestStream().Write(buffer, 0, buffer.Length); } ////设置最大连接 //if (item.Connectionlimit > 0) //{ // request.ServicePoint.ConnectionLimit = item.Connectionlimit; //} return(request); }