static void Main(string[] args)
{
IHttpProvider httpProvider = new HttpProvider();
// 1. 模拟一个Get请求方式
HttpResponseParameter responseParameter1 = httpProvider.Excute(new HttpRequestParameter
{
Url = "http://www.baidu.com",
IsPost = false,
Encoding = Encoding.UTF8
//Cookie = new HttpCookieType() 如果需要Cookie
});
System.Console.WriteLine(responseParameter1.Body);
// 2. 模拟一个Post请求方式（例：登录)
IDictionary <string, string> postData = new Dictionary <string, string>();
postData.Add("userName", "登录用户名");
postData.Add("userPwd", "用户名密码");
HttpResponseParameter responseParameter2 = httpProvider.Excute(new HttpRequestParameter
{
Url = "你的登录url",
IsPost = true,
Encoding = Encoding.UTF8,
Parameters = postData
});
System.Console.WriteLine(responseParameter2.Body);
System.Console.ReadLine();
}
protected void Login_Click(object sender, RoutedEventArgs e)
{
var userName = UserName.Text.Trim();
var passWord = PassWord.Password.Trim();
var code = VerCode.Text.Trim();
var dicPostData = new Dictionary <string, string>();
dicPostData.Add("client", "www");
dicPostData.Add("referer", HttpUtility.UrlEncode("http://www.jikexueyuan.com/"));
dicPostData.Add("uname", userName);
dicPostData.Add("password", passWord);
dicPostData.Add("verify", code);
HttpResponseParameter responseParameter = HttpProvider.Excute(new HttpRequestParameter
{
Url = "http://passport.jikexueyuan.com/submit/login?is_ajax=1",
IsPost = true,
Parameters = dicPostData,
Cookie = SessionCookie
});
LoginResultEntity loginResult = responseParameter.Body.DeserializeObject <LoginResultEntity>();
if (loginResult.status == 1)
{
// 2.登录成功，保存cookie
// CookieStoreInstance.CurrentCookie = responseParameter.Cookie;
}
}
/// <summary>
/// 获取登录的Cookie
/// </summary>
void GetLoginSessionCookie()
{
var httpResponse = HttpProvider.Excute(new HttpRequestParameter()
{
Url = "https://kyfw.12306.cn/otn/login/init",
IsPost = false
});
SessionCookie = httpResponse.Cookie;
}
/// <summary>
/// 获取登录的Cookie
/// </summary>
void GetLoginSessionCookie()
{
var httpResponse = HttpProvider.Excute(new HttpRequestParameter()
{
Url = "http://passport.jikexueyuan.com/sso/login",
IsPost = false
});
SessionCookie = httpResponse.Cookie;
}
static void Main(string[] args)
{
IHttpProvider httpProvider = new HttpProvider();
// 1. 模拟一个Get请求方式
//HttpResponseParameter responseParameter1 = httpProvider.Excute(new HttpRequestParameter
//{
// Url = "http://www.baidu.com",
// IsPost = false,
// Encoding = Encoding.UTF8
// //Cookie = new HttpCookieType() 如果需要Cookie
//});
//System.Console.WriteLine(responseParameter1.Body);
// 2. 模拟一个Post请求方式（例：登录)
//IDictionary<string, string> postData = new Dictionary<string, string>();
//postData.Add("userName", "登录用户名");
//postData.Add("userPwd", "用户名密码");
//HttpResponseParameter responseParameter2 = httpProvider.Excute(new HttpRequestParameter
//{
// Url = "你的登录url",
// IsPost = true,
// Encoding = Encoding.UTF8,
// Parameters = postData
//});
//System.Console.WriteLine(responseParameter2.Body);
// 1. 模拟一个Get请求方式
//http://lbfjapi.wsdict.com:8000/xpapi.ashx?method=xpshop.trades.sold.get&page_no=1&page_size=100
//3.模拟一个Get请求方式 带参数
IDictionary <string, string> postData = new Dictionary <string, string>();
postData.Add("userName", "登录用户名");
postData.Add("userPwd", "用户名密码");
HttpRequestParameter HttpReq = new HttpRequestParameter
{
//Url = "http://www.baidu.com",
Url = "http://lbfjapi.wsdict.com:8000/xpapi.ashx?method=xpshop.shipping.type.get",
IsPost = false,
Encoding = Encoding.UTF8,
Parameters = postData
//Cookie = new HttpCookieType() 如果需要Cookie
};
System.Console.WriteLine("请求数据为：" + HttpReq.ToString());
HttpResponseParameter responseParameter3 = httpProvider.Excute(HttpReq);
System.Console.WriteLine(responseParameter3.Body);
System.Console.ReadLine();
}
void PostUserLogin()
{
var dicPostData2 = new Dictionary <string, string>();
dicPostData2.Add("_json_att", "");
HttpResponseParameter responseParameter2 = HttpProvider.Excute(new HttpRequestParameter
{
Url = "https://kyfw.12306.cn/otn/login/userLogin",
IsPost = true,
Parameters = dicPostData2,
Cookie = CookieStoreInstance.CurrentCookie
});
}
void CheckUserLogin()
{
var dicPostData2 = new Dictionary <string, string>();
dicPostData2.Add("_json_att", "");
HttpResponseParameter responseParameter2 = HttpProvider.Excute(new HttpRequestParameter
{
Url = "https://kyfw.12306.cn/otn/login/checkUser",
IsPost = true,
Parameters = dicPostData2,
Cookie = SessionCookie
});
}
public void TestMethod1()
{
var httpProvider = new HttpProvider();
var url =
string.Format(
"https://kyfw.12306.cn/otn/leftTicket/queryT?leftTicketDTO.train_date={0}&leftTicketDTO.from_station={1}&leftTicketDTO.to_station={2}&purpose_codes={3}", "2016-02-04", "SHH", "FYH", "ADULT");
HttpResponseParameter responseParameter = httpProvider.Excute(new HttpRequestParameter
{
Url = url,
IsPost = false
});
HttpJsonResult <QeryTicketsData> loginResult = responseParameter.Body.DeserializeObject <HttpJsonResult <QeryTicketsData> >();
}
void CheckRandCode(string randCode)
{
var dicPostData = new Dictionary <string, string>();
dicPostData.Add("rand", "sjrand");
dicPostData.Add("randCode", randCode);
HttpResponseParameter responseParameter = HttpProvider.Excute(new HttpRequestParameter
{
Url = "https://kyfw.12306.cn/otn/passcodeNew/checkRandCodeAnsyn",
IsPost = true,
Parameters = dicPostData,
Cookie = SessionCookie
});
}
void ShowCode()
{
GetLoginSessionCookie();
var request = new HttpRequestParameter();
request.Url = string.Format("https://kyfw.12306.cn/otn/passcodeNew/getPassCodeNew?module=login&rand=sjrand&{0}",
DateTime.Now.ToString("yyyyMMddHHmmsss"));
request.ResponseEnum = HttpResponseEnum.Stream;
request.Cookie = SessionCookie;
request.StreamAction = x =>
{
MemoryStream ms = new MemoryStream();
byte[] buffer = new byte[1024];
while (true)
{
int sz = x.Read(buffer, 0, buffer.Length);
if (sz == 0)
{
break;
}
ms.Write(buffer, 0, sz);
}
ms.Position = 0;
BitmapImage bmp = new BitmapImage();
bmp.BeginInit();
bmp.StreamSource = new MemoryStream(ms.ToArray());
bmp.EndInit();
viefCode.Source = bmp;
ms.Close();
};
var response = HttpProvider.Excute(request);
SessionCookie.CookieCollection.Add(response.Cookie.CookieCollection);
SessionCookie.CookieString = SessionCookie.CookieString + "," + response.Cookie.CookieString;
}
public string OperateCallBaceResult(object requestParameters, string operateUrl, bool error)
{
try
{
string body = Newtonsoft.Json.JsonConvert.SerializeObject(requestParameters);
string timestamp = OperateUtil.GetTimeStamp().ToString();
string nonce = OperateUtil.GetNonceString(8);
String sign = SignUtil.sign(this.merchantKey, timestamp, nonce, body);
Dictionary <string, string> map = new Dictionary <string, string>();
if (error)
{
sign = "12312312qwed";
}
map.Add("body", body);
map.Add("sign", sign);
map.Add("timestamp", timestamp);
map.Add("nonce", nonce);
Dictionary <string, string> Spaheader = new Dictionary <string, string>();
Dictionary <string, string> myParameters = new Dictionary <string, string>();
HttpProvider httpProvider = new HttpProvider();
HttpResponseParameter reponseParameter = httpProvider.Excute(new HttpRequestParameter
{
Url = operateUrl,
IsPost = true,
Encoding = Encoding.UTF8,
Parameters = map,
HeaderParameters = map,
}, "", "");
string textReg = reponseParameter.Body;
return(textReg);
}
catch (Exception ex)
{
throw ex;
}
}
/// <summary>
/// 显示验证码
/// </summary>
void ViewCode()
{
GetLoginSessionCookie();
var request = new HttpRequestParameter();
request.Url = string.Format("http://passport.jikexueyuan.com/sso/verify?t={0}",
DateTime.Now.ToString("yyyyMMddHHmmsss"));
request.ResponseEnum = HttpResponseEnum.Stream;
request.Cookie = SessionCookie;
request.StreamAction = x =>
{
MemoryStream ms = new MemoryStream();
byte[] buffer = new byte[1024];
while (true)
{
int sz = x.Read(buffer, 0, buffer.Length);
if (sz == 0)
{
break;
}
ms.Write(buffer, 0, sz);
}
ms.Position = 0;
BitmapImage bmp = new BitmapImage();
bmp.BeginInit();
bmp.StreamSource = new MemoryStream(ms.ToArray());
bmp.EndInit();
VerifyCode.Source = bmp;
ms.Close();
};
HttpProvider.Excute(request);
}
/// <summary>
/// 开始爬取高手操作数据
/// </summary>
/// <param name="pageNum"></param>
/// <param name="loopTime"></param>
public static Message Excute(int pageNum = 1, int loopTime = 0)
{
System.GC.Collect();
LogUtility.LogAction(string.Format(@"爬取第{0}页", pageNum));
isStopPoint = false;
string _refererUrl = "http://group.eastmoney.com/Master.html?type=GSCZ";
string url = "http://group.eastmoney.com/findmaster{0}.html";
IHttpProvider httpProvider = new HttpProvider();
url = string.Format(url, pageNum);
//Get请求方式
HttpResponseParameter responseParameter1 = httpProvider.Excute(new HttpRequestParameter
{
Url = url,
RefererUrl = _refererUrl,
IsPost = false,
Encoding = Encoding.UTF8
});
string htmlContent = responseParameter1.Body;
HtmlDocument htmlDoc = new HtmlDocument();
htmlDoc.LoadHtml(htmlContent);
string nodeName = "//tbody";
HtmlNode tbodyNode = htmlDoc.DocumentNode.SelectSingleNode(nodeName);
if (tbodyNode != null && !string.IsNullOrWhiteSpace(tbodyNode.InnerText.Trim()))
{
bool isSave = AnalysisHtml(htmlDoc, pageNum);
#region 入库失败，重试
if (!isSave && loopTime <= 3)
{
Thread.Sleep(5000);
LogUtility.LogAction(string.Format(@"入库失败，重新爬取第{0}次", loopTime + 1));
Excute(pageNum, loopTime + 1);
}
if (loopTime > 3)
{
LogUtility.LogAction(string.Format(@"已重新爬取{0}次,退出重试", loopTime + 1));
}
#endregion
if (!isStopPoint)
{
Excute(pageNum + 1);
}
}
else
{
LogUtility.LogAction("爬取到最后一页，从头开始爬取");
//加载停止点
EastMoneyMain.InitInfo();
//从第一页开始爬取
Excute();
}
return(new Message());
}
protected void btn_Click(object sender, RoutedEventArgs e)
{
var userName = UserName.Text.Trim();
var passWord = PassWord.Password.Trim();
var code = string.Empty;
if (RandCodeImage1.IsChecked == true)
{
code += "32,53";
}
if (RandCodeImage2.IsChecked == true)
{
code += ",103,45";
}
if (RandCodeImage3.IsChecked == true)
{
code += ",178,55";
}
if (RandCodeImage4.IsChecked == true)
{
code += ",243,44";
}
if (RandCodeImage5.IsChecked == true)
{
code += ",45,113";
}
if (RandCodeImage6.IsChecked == true)
{
code += ",107,114";
}
if (RandCodeImage7.IsChecked == true)
{
code += ",172,98";
}
if (RandCodeImage8.IsChecked == true)
{
code += ",249,117";
}
code = code.Trim(',');
CheckRandCode(code);
var dicPostData = new Dictionary <string, string>();
dicPostData.Add("loginUserDTO.user_name", userName);
dicPostData.Add("userDTO.password", passWord);
dicPostData.Add("randCode", code);
HttpResponseParameter responseParameter = HttpProvider.Excute(new HttpRequestParameter
{
Url = "https://kyfw.12306.cn/otn/login/loginAysnSuggest",
IsPost = true,
Parameters = dicPostData,
Cookie = SessionCookie
});
HttpJsonResultEntity <UserLoginResult> loginResut = JsonConvert.DeserializeObject <HttpJsonResultEntity <UserLoginResult> >(responseParameter.Body);
SessionCookie.CookieCollection.Add(responseParameter.Cookie.CookieCollection);
SessionCookie.CookieString = SessionCookie.CookieString + "," +
responseParameter.Cookie.CookieString;
CookieStoreInstance.CurrentCookie = SessionCookie;
this.Close();
//PostUserLogin();
//CheckUserLogin();
//var dicPostData2 = new Dictionary<string, string>();
//dicPostData.Add("_json_att", "");
//dicPostData.Add("REPEAT_SUBMIT_TOKEN", "b4ed83a3647be9dfb24f3fbfe8f351b5");
//HttpResponseParameter responseParameter2 = HttpProvider.Excute(new HttpRequestParameter
//{
// Url = "https://kyfw.12306.cn/otn/confirmPassenger/getPassengerDTOs",
// IsPost = true,
// Parameters = dicPostData2,
// Cookie = SessionCookie
//});
//HttpJsonResultEntity<QueryPassengerResult> passengerResult = JsonConvert.DeserializeObject<HttpJsonResultEntity<QueryPassengerResult>>(responseParameter2.Body);
//LoginResultEntity loginResult = responseParameter.Body.DeserializeObject<LoginResultEntity>();
//if (loginResult.status == 1)
//{
// // 2.登录成功，保存cookie
// // CookieStoreInstance.CurrentCookie = responseParameter.Cookie;
}