/// <summary>
/// GET方式调用Web Api
/// </summary>
/// <param name="baseUrl">基准URL（不含查询串）</param>
/// <param name="dictParam">查询参数字典</param>
/// <param name="result">返回数据（Json格式）</param>
/// <param name="errMsg">出错信息</param>
/// <param name="code">返回状态码</param>
/// <param name="CookieName">cookie名称</param>
/// <returns>成功与否</returns>
public static bool Get(string baseUrl, Dictionary <string, string> dictParam,
out string result, out string errMsg, out HttpStatusCode code, string CookieName)
{
errMsg = string.Empty;
result = string.Empty;
try
{
HttpHelper http = new HttpHelper();
HttpItem item = new HttpItem();
var url = "";
if (dictParam == null)
{
url = GetLastUrl(baseUrl, new Dictionary <string, string>());
}
else
{
url = GetLastUrl(baseUrl, dictParam);
}
var AuthorizationValue = "";
AuthorizationValue = Utils.GetCookie(CookieName);
item.URL = url;
item.Method = "GET";
item.ContentType = "application/json";//返回类型 可选项有默认值
if (AuthorizationValue != null && AuthorizationValue != "null" && AuthorizationValue != "")
{
//发送头部身份验证请求
item.Header.Add("Authorization", "Bearer " + AuthorizationValue);
}
if (CookieName == BaseGlobalParameter.defUinfoAdminCookie) //如果是后台发送的
{
item.Header.Add("AdminNoUserSingle", "true");
}
HttpResult GetData = http.GetHtml(item);
if (GetData.StatusCode == HttpStatusCode.OK)
{
result = GetData.Html;
errMsg = "";
code = GetData.StatusCode;
return(true);
}
if (GetData.StatusCode == HttpStatusCode.Unauthorized || GetData.StatusCode == HttpStatusCode.Forbidden)
{
errMsg = "该用户未授权";
code = GetData.StatusCode;
return(false);
}
code = GetData.StatusCode;
return(false);
}
catch (Exception ex)
{
errMsg = ex.Message;
code = HttpStatusCode.InternalServerError;
return(false);
}
}
protected string Test(string ip)
{
// ip = ip.Replace(":", "：");
HttpHelper http = new HttpHelper();
ServicePointManager.DefaultConnectionLimit = 1000;
HttpItem item = new HttpItem()
{
URL = "https://www.taobao.com/",//URL 必需项
Encoding = null,//编码格式（utf-8,gb2312,gbk） 可选项 默认类会自动识别
Method = "get",//URL 可选项 默认为Get
Timeout = 250000,//连接超时时间 可选项默认为100000
ReadWriteTimeout = 30000,//写入Post返回结果超时时间 可选项默认为30000
KeepAlive = false,
ProtocolVersion = HttpVersion.Version10,
IsToLower = false,//得到的HTML代码是否转成小写 可选项默认转小写
//Cookie = "",//字符串Cookie 可选项
//UserAgent = "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0)",//用户的浏览器类型，版本，操作系统 可选项有默认值
//Accept = "text/html, application/xhtml+xml, */*",// 可选项有默认值
ContentType = "text/html",//返回类型 可选项有默认值
Referer = "http://www.ip.cn/",//来源URL 可选项
CerPath = "taobao.cer",
ResultType = ResultType.Byte ,
//Allowautoredirect = true,//是否根据３０１跳转 可选项
Connectionlimit = 1024,//最大连接数 可选项 默认为1024
ProxyIp = ip,//代理服务器ID 端口可以直接加到后面以：分开就行了 可选项 不需要代理 时可以不设置这三个参数
// ResultType = ResultType.String,//返回返回结果类型，是Byte还是String
CookieCollection = new System.Net.CookieCollection(),//可以直接传一个Cookie集合进来
};
//item.Header.Add("测试Key1", "测试Value1");
//item.Header.Add("测试Key2", "测试Value2");
//得到HTML代码
HttpResult result = http.GetHtml(item);
//取出返回的Cookie
string cookie = result.Cookie;
//返回的Html内容
string html = result.Html;
if (result.StatusCode == System.Net.HttpStatusCode.OK)
{
}
//表示StatusCode的文字说明与描述
string statusCodeDescription = result.StatusDescription;
return html;
}
/// <summary>
/// POST方式调用Web Api
/// </summary>
/// <param name="baseUrl">基准URL（不含查询串）</param>
/// <param name="dictParam">查询参数字典</param>
/// <param name="parseData">传递实体数据（Json格式）</param>
/// <param name="result">返回数据（Json格式）</param>
/// <param name="errMsg">出错信息</param>
/// <param name="code">返回状态码</param>
/// <param name="CookieName">cookie名称</param>
/// <returns>成功与否</returns>
public static bool Post(string baseUrl, Dictionary <string, string> dictParam,
out string result, out string errMsg, out HttpStatusCode code, string CookieName)
{
errMsg = string.Empty;
result = string.Empty;
try
{
HttpHelper http = new HttpHelper();
HttpItem item = new HttpItem();
var strjson = "";
if (dictParam == null)
{
strjson = "";
}
else
{
strjson = SerializeDictionaryToJsonString(dictParam); //请求参数
}
var AuthorizationValue = "";
AuthorizationValue = Utils.GetCookie(CookieName);
item.URL = baseUrl;
item.Method = "POST";
if (strjson == "")
{
item.Postdata = "{}";
}
else
{
item.Postdata = strjson;
}
item.PostDataType = PostDataType.String;
item.PostEncoding = System.Text.Encoding.UTF8;
item.ContentType = "application/json"; //返回类型 可选项有默认值
if (AuthorizationValue != null && AuthorizationValue != "null" && AuthorizationValue != "")
{ //发送头部身份验证请求
item.Header.Add("Authorization", "Bearer " + AuthorizationValue);
}
if (CookieName == BaseGlobalParameter.defUinfoAdminCookie) //如果是后台发送的
{
item.Header.Add("AdminNoUserSingle", "true");
}
HttpResult GetData = http.GetHtml(item);
if (GetData.StatusCode == HttpStatusCode.OK)
{
result = GetData.Html;
errMsg = "";
code = GetData.StatusCode;
return(true);
}
if (GetData.StatusCode == HttpStatusCode.Unauthorized || GetData.StatusCode == HttpStatusCode.Forbidden)
{
errMsg = "该用户未授权";
code = GetData.StatusCode;
return(false);
}
code = GetData.StatusCode;
return(false);
}
catch (Exception ex)
{
errMsg = ex.Message;
code = HttpStatusCode.InternalServerError;
return(false);
}
}
static void Main(string[] args)
{
//List<Thread> threads = new List<Thread>();
while (true)
{
Console.WriteLine("请输入关键词：");
String keyWord = Console.ReadLine().Trim();
if (!string.IsNullOrEmpty(keyWord.Trim()))
{
//
Thread t = new Thread(() =>
{
#region 抓取任务开始
String start = "";
string taskKeyWord = keyWord;
HttpHelper http = new HttpHelper();
//每一次抓取
//大于totalItems/48时停止
Int32 count = 1;
Random random = new Random();
while (true)
{
start = (48 * count).ToString();
HttpItem item = new HttpItem()
{
URL = "http://pic.sogou.com/pics?query=" + taskKeyWord + "&mood=0&picformat=0&mode=1&di=0&clusterfilter=off&p=40230504&dp=1&start=" + start + "&reqType=ajax&tn=0&reqFrom=result",//URL 必需项
Method = "get",//URL 可选项 默认为Get
IsToLower = false,//得到的HTML代码是否转成小写 可选项默认转小写
Referer = "",//来源URL 可选项
Postdata = "",//Post数据 可选项GET时不需要写
Timeout = 100000,//连接超时时间 可选项默认为100000
ReadWriteTimeout = 30000,//写入Post数据超时时间 可选项默认为30000
UserAgent = "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0)",//用户的浏览器类型，版本，操作系统 可选项有默认值
ContentType = "text/html",//返回类型 可选项有默认值
Allowautoredirect = false,//是否根据301跳转 可选项
//CerPath = "d:\123.cer",//证书绝对路径 可选项不需要证书时可以不写这个参数
//Connectionlimit = 1024,//最大连接数 可选项 默认为1024
ResultType = ResultType.String
};
HttpResult result = http.GetHtml(item);
string html = result.Html;
string cookie = result.Cookie;
//Console.WriteLine("【" + Thread.CurrentThread.ManagedThreadId + "】-" + html);
//解析结果，得到停止条件
System.Web.Script.Serialization.JavaScriptSerializer js = new System.Web.Script.Serialization.JavaScriptSerializer();
SouGouImgModel model = js.Deserialize<SouGouImgModel>(html);
if (Convert.ToInt32(start) >= Convert.ToInt32(model.totalItems.Replace(",", "")))
//if (model.items.Count==0)
{
Console.WriteLine("结束时间：【" + DateTime.Now + "】-任务关键字：" + taskKeyWord);
break;
}
Console.WriteLine("当前个数："+model.items.Count+"==请求总个数："+start+"==MaxEnd："+model.maxEnd+"==总个数："+model.totalItems);
//下一轮
int span = random.Next(3000, 10000);
Thread.Sleep(span);
//Console.WriteLine(count);
count++;
}
#endregion
});
//threads.Add(t);
t.Start();
Console.WriteLine("开始时间：【" + DateTime.Now + "】-任务关键字：" + keyWord);
}
}
}