public virtual List <IPProxy> GetIPProxyListAll()
{
try
{
Database database = DatabaseFactory.CreateDatabase("DatabaseProxy");
DbCommand dbCommand = database.GetStoredProcCommand("PR_IPProxy_SelectAll");
//database.AddInParameter(dbCommand, "@TickerList", DbType.String, tickerList.ToString());
List <IPProxy> listItem = new List <IPProxy>();
using (IDataReader reader = database.ExecuteReader(dbCommand))
{
while (reader.Read())
{
IPProxy item = CreateShareTypeFromReader(reader);
listItem.Add(item);
}
reader.Close();
}
//totalRecords = (int)database.GetParameterValue(dbCommand, "@TotalRecords");
return(listItem);
}
catch (Exception ex)
{
// log this exception
log4net.Util.LogLog.Error(ex.Message, ex);
// wrap it and rethrow
throw;
}
}
/// <summary>
/// REFERENCES:
/// other approach:
///
/// http://stackoverflow.com/questions/18921099/add-proxy-to-phantomjsdriver-selenium-c
/// PhantomJSOptions phoptions = new PhantomJSOptions();
/// phoptions.AddAdditionalCapability(CapabilityType.Proxy, "http://localhost:9999");
///
/// driver = new PhantomJSDriver(phoptions);
/// - (24/04/2019) https://stackoverflow.com/a/52446115/3796898
///
/// </summary>
/// <param name="hostPgUrlPattern"></param>
/// <param name="freeProxy"></param>
/// <returns></returns>
public static string LoadIPProxyDocumentContent(string hostPgUrlPattern, IPProxy freeProxy = null)
{
//var options = new ChromeOptions();
//var userAgent = "user_agent_string";
//options.AddArgument("--user-agent=" + userAgent);
//IWebDriver driver = new ChromeDriver(options);
var service = PhantomJSDriverService.CreateDefaultService(".\\");
service.HideCommandPromptWindow = true;
if (freeProxy != null)
{
service.ProxyType = freeProxy.Protocol == ProxyProtocolsEnum.HTTP ? "http" : "https:";
var proxy = new Proxy
{
HttpProxy = $"{freeProxy.IPAddress}:{freeProxy.PortNo}",
};
service.Proxy = proxy.HttpProxy;
}
var driver = new PhantomJSDriver(service)
{
Url = hostPgUrlPattern
};
driver.Navigate();
var content = driver.PageSource;
driver.Quit();
return(content);
}
/// <summary>
/// ip无效处理
/// </summary>
private void IPInvalidProcess(IPProxy ipproxy)
{
Settings.SetUnviableIP(ipproxy);//设置为无效代理
if (ipproxy != null)
{
DBChangeQueue.Instance.EnQueue(new StorageData()
{
Name = "IPProxy",
// Document = new BsonDocument().Add("status", "1"),
Document = new BsonDocument().Add(string.Format("{0}_status", DataTableName), "1"),
Query = Query.EQ("ip", ipproxy.IP),
Type = StorageType.Update
});
StartDBChangeProcess();
}
if (!string.IsNullOrEmpty(Settings.LoginAccount))
{
DBChangeQueue.Instance.EnQueue(new StorageData()
{
Name = "QCCAccount",
// Document = new BsonDocument().Add("status", "1"),
Document = new BsonDocument().Add(string.Format("status", DataTableName), "1"),
Query = Query.EQ("name", Settings.LoginAccount),
Type = StorageType.Update
});
StartDBChangeProcess();
//验证验证码重新进行设定
}
}
public HttpResponseMessage Post(ParaProxy paraProxy)
{
if (ModelState.IsValid)
{
#region insert database here
try
{
IPProxy ipProxy = new IPProxy {
IPAddress = paraProxy.IPAddress, IPPort = int.Parse(paraProxy.IPPort), StatusIP = bool.Parse(paraProxy.StatusIP), CreateDate = int.Parse(paraProxy.CreateDate)
};
IPProxyServices.CreateIPProxy(ipProxy);
}
catch (Exception)
{
return(Request.CreateErrorResponse(HttpStatusCode.BadRequest, ModelState));
}
#endregion end
HttpResponseMessage response = Request.CreateResponse(HttpStatusCode.Created, paraProxy);
return(response);
}
else
{
return(Request.CreateErrorResponse(HttpStatusCode.BadRequest, ModelState));
}
}
public void UpdateProxy(IPProxy proxy)
{
if (proxy != null)
{
var script = $"return phantom.setProxy(\"{proxy.IPAddress}\", {proxy.PortNo}, \"http\", \"\", \"";
var obj = (this.WebDriver as PhantomJSDriver)?.ExecutePhantomJS(script);
}
}
/// <summary>
///
/// </summary>
/// <param name="proxy"></param>
protected void RegisterProxy(IPProxy proxy)
{
AgentStatus = IPProxyAgentStatusEnum.Parsing;
lock (IPProxies)
{
IPProxies.Add(proxy);
}
InvokeEventFreeIPProxyParsed(new EventHandlers.FreeIPProxyParsedEventArgs(PageNo, TargetPgUrl, proxy));
}
public static bool TestIPProxy(IPProxy proxy)
{
var protHt = proxy.Protocol == IPProxyRules.ProxyProtocolsEnum.HTTPS ? "https://" : proxy.Protocol == IPProxyRules.ProxyProtocolsEnum.HTTP ? "http://" : "";
var prxy = new WebProxy()
{
Address = new Uri($"{protHt}{proxy.IPAddress}:{proxy.PortNo}"),
Credentials = CredentialCache.DefaultCredentials,
//still use the proxy for local addresses
BypassProxyOnLocal = false,
};
if (proxy.Credential != null)
{
prxy.UseDefaultCredentials = false;
// *** These credentials are given to the proxy server, not the web server ***
prxy.Credentials = proxy.ToNetworkCredential();
}
var sw = Stopwatch.StartNew();
// Finally, create the HTTP client object to test the proxy
var request = (HttpWebRequest)WebRequest.Create("http://example.com/");
request.Proxy = prxy;
try
{
proxy.CheckStatus = IPProxy.CheckStatusEnum.Checking;
using (var req = request.GetResponse())
{
using (var reader = new StreamReader(req.GetResponseStream()))
{
var response = reader.ReadToEnd();
proxy.CheckStatus = IPProxy.CheckStatusEnum.Checked;
proxy.SpeedRate = (int)sw.ElapsedMilliseconds;
}
}
}
catch (Exception e)
{
proxy.CheckStatus = IPProxy.CheckStatusEnum.CheckedInvalid;
proxy.SpeedRate = 0;
}
finally
{
sw.Stop();
proxy.LastChecked = DateTime.Now;
}
return(proxy.CheckStatus == IPProxy.CheckStatusEnum.Checked);
}
public static void CreateIPProxy(IPProxy iPProxy)
{
try
{
iPProxyDao = new IPProxyDao();
iPProxyDao.CreateIPProxy(iPProxy);
//return aGNews_ArticlesDAO.
}
catch (Exception)
{
throw;
}
}
/// <summary>
/// ip无效处理
/// </summary>
private void IPInvalidProcess(IPProxy ipproxy)
{
Settings.SetUnviableIP(ipproxy);//设置为无效代理
if (ipproxy != null)
{
DBChangeQueue.Instance.EnQueue(new StorageData()
{
Name = "IPProxy",
Document = new BsonDocument().Add("status", "1"),
Query = Query.EQ("ip", ipproxy.IP)
});
StartDBChangeProcess();
}
}
private void IPInvalidProcess(IPProxy ipproxy)
{
this.Settings.SetUnviableIP(ipproxy);
if (ipproxy != null)
{
StorageData target = new StorageData {
Name = "IPProxy",
Document = new BsonDocument().Add("status", "1"),
Query = Query.EQ("ip", ipproxy.IP)
};
DBChangeQueue.Instance.EnQueue(target);
this.StartDBChangeProcess();
}
}
/// <summary>
/// ip无效处理
/// </summary>
private static void IPInvalidProcess(IPProxy ipproxy)
{
Settings.SetUnviableIP(ipproxy); //设置为无效代理
simpleCrawler.SimulateLogin(); //模拟登陆
if (ipproxy != null)
{
DBChangeQueue.Instance.EnQueue(new StorageData()
{
Name = "IPProxy",
Document = new BsonDocument().Add(string.Format("{0}_status", simpleCrawler.DataTableName), "1"),
Query = Query.EQ("ip", ipproxy.IP)
});
StartDBChangeProcess();
}
}
public static bool TestIPProxy2(IPProxy proxy)
{
try
{
var sw = Stopwatch.StartNew();
proxy.CheckStatus = IPProxy.CheckStatusEnum.Checking;
var content = SeleniumHelper.GrabPage("http://example.com", proxy.AsTuple());
sw.Stop();
proxy.SpeedRate = (int)sw.ElapsedMilliseconds;
proxy.CheckStatus = IPProxy.CheckStatusEnum.Checked;
}
catch (Exception e)
{
proxy.CheckStatus = IPProxy.CheckStatusEnum.CheckedInvalid;
}
return(proxy.CheckStatus == IPProxy.CheckStatusEnum.Checked);
}
/// <summary>
///
/// </summary>
/// <param name="proxy"></param>
public void GrabFirstOrNextPage(IPProxy proxy = null)
{
if (AgentStatus != IPProxyAgentStatusEnum.Idle)
{
return;
}
var pgNo = PageNo.ToString();
if (PageInstruction != null)
{
pgNo = PageInstruction.PageNo(PageNo);
}
TargetPgUrl = TargetPageUrlPattern.Replace("{PAGENO}", pgNo);
AgentStatus = IPProxyAgentStatusEnum.Reading;
InvokeEventFreeIPProxiesReading(new EventHandlers.FreeIPProxiesReadingEventArgs(TargetPgUrl));
// Get the html response called from the url
var contentDoc = ScraperBoxHelper.GrabPage(_driver as PhantomJSDriver, TargetPgUrl, proxy?.AsTuple());
ValidatePage(contentDoc);
if (!PageIsValid)
{
AgentStatus = IPProxyAgentStatusEnum.Completed;
InvokeEventFreeIPProviderSourceCompleted(new EventHandlers.FreeIPProviderSourceCompletedEventArgs(PageNo));
base.Shutdown();
return;
}
// have the inheriting class parse the html result
ParseProxyPage(contentDoc);
TotalPagesScraped++;
// increase the page no for next use.
PageNo++;
}
public virtual void CreateIPProxy(IPProxy iPProxy)
{
try
{
Database database = DatabaseFactory.CreateDatabase("DatabaseProxy");
DbCommand dbCommand = database.GetStoredProcCommand("PR_IPProxy_Insert");
database.AddInParameter(dbCommand, "@IPAddress", DbType.AnsiString, iPProxy.IPAddress);
database.AddInParameter(dbCommand, "@IPPort", DbType.Int32, iPProxy.IPPort);
database.AddInParameter(dbCommand, "@StatusIP", DbType.Boolean, iPProxy.StatusIP);
database.AddInParameter(dbCommand, "@CreateDate", DbType.Int32, iPProxy.CreateDate);
database.AddOutParameter(dbCommand, "@Id", DbType.Int32, iPProxy.Id);
database.ExecuteNonQuery(dbCommand);
iPProxy.Id = (int)database.GetParameterValue(dbCommand, "@Id");
}
catch (Exception ex)
{
// log this exception
log4net.Util.LogLog.Error(ex.Message, ex);
// wrap it and rethrow
throw new ApplicationException();
}
}
public virtual IPProxy CreateShareTypeFromReader(IDataReader reader)
{
IPProxy item = new IPProxy();
try
{
if (!reader.IsDBNull(reader.GetOrdinal("Id")))
{
item.Id = (int)reader["Id"];
}
if (!reader.IsDBNull(reader.GetOrdinal("IPAddress")))
{
item.IPAddress = (string)reader["IPAddress"];
}
if (!reader.IsDBNull(reader.GetOrdinal("IPPort")))
{
item.IPPort = (int)reader["IPPort"];
}
if (!reader.IsDBNull(reader.GetOrdinal("StatusIP")))
{
item.StatusIP = (bool)reader["StatusIP"];
}
if (!reader.IsDBNull(reader.GetOrdinal("CreateDate")))
{
item.CreateDate = (int)reader["CreateDate"];
}
}
catch (Exception ex)
{
// log this exception
log4net.Util.LogLog.Error(ex.Message, ex);
// wrap it and rethrow
throw;
}
return(item);
}
/// <summary>
/// 模拟登陆，ip代理可能需要用到
/// </summary>
/// <returns></returns>
public bool SimulateLogin()
{
//return true;
//Settings.SimulateCookies = "pgv_pvid=1513639250; aliyungf_tc=AQAAADPRHnGrvwQAIkg9OwOqtkYJbU4N; oldFlag=1; CNZZDATA1259577625=112366950-1466409958-%7C1466415358; hide-index-popup=1; hide-download-panel=1; _alicdn_sec=576ba5f9a986fb4802dacf51bc99b1e76724f58e; connect.sid=s%3AeYWXycPKai63BYTmB9d6h-0IM_R2kp6n.EUgfW0AmJ6GB%2F0TamTi4tT53QK4OR4yQtU1I3Ba8Ryo; userKey=QXBAdmin-Web2.0_N3iUdNobAoys4M395Pk5v%2F6Zxcwjt1tiCqeSf3X3ZnI%3D; userValue=bea26f0d-e414-168a-0fe2-b8eb4278ab07; Hm_lvt_52d64b8d3f6d42a2e416d59635df3f71=1464663982,1464775028,1464776749,1465799273; Hm_lpvt_52d64b8d3f6d42a2e416d59635df3f71=1466672591";//设置cookie值
//return true;
if (!string.IsNullOrEmpty(Settings.LoginAccount))
{
return(true);
}
// return SimulateLoginEx();
var userName = string.Empty;
var passWord = string.Empty;
if (AccountQueue.Count() > 0)
{
var _curCookie = AccountQueue.Dequeue();
if (_curCookie != null)
{
Console.WriteLine("提取账号{0}", _curCookie.Text("name"));
userName = _curCookie.Text("name");
passWord = _curCookie.Text("password");
Settings.LoginAccount = userName;
}
else
{
Environment.Exit(0);
}
}
else
{
Environment.Exit(0);
}
return(AliyunAutoLoin(userName, passWord));
IPProxy ipProxy = null;
HttpManager.Instance.InitWebClient(hi, true, 30, 30);
Random rand = new Random(Environment.TickCount);
var nameNormal = userName;
var pwdNormal = passWord;
if (string.IsNullOrEmpty(nameNormal) || string.IsNullOrEmpty(pwdNormal))
{
return(false);
}
var validUrl = "";
var postFormat = "";
bool result = false;
// var postFormat = "geetest_challenge={0}&geetest_validate={1}&geetest_seccode={1}%7Cjordan&requestType=search_enterprise";
var passResult = geetestHelper.PassGeetest(hi, postFormat, validUrl, Settings.SimulateCookies);
result = passResult.Status;
// this.richTextBoxInfo.Document.Blocks.Clear();
//this.webBrowser.Refresh();
if (passResult.Status)
{
hi.Url = "http://www.qichacha.com/user_loginaction";
hi.Refer = "http://www.qichacha.com/user_login";
hi.PostData = string.Format("nameNormal={0}&pwdNormal={1}&geetest_challenge={2}&geetest_validate={3}&geetest_seccode={3}%7Cjordan", nameNormal, pwdNormal, passResult.Challenge, passResult.ValidCode);
var ho = HttpManager.Instance.ProcessRequest(hi);
if (ho.IsOK)
{
if (ho.TxtData.Contains("true"))
{
Settings.SimulateCookies = ho.Cookies;
Console.WriteLine("过验证码模拟登陆成功");
return(true);
}
}
var resultText = geetestHelper.GetLastPoint(hi);
Console.WriteLine(resultText);
}
return(false);
}
/// <summary>
/// 模拟登陆，ip代理可能需要用到
/// </summary>
/// <returns></returns>
public bool SimulateLogin()
{
return(true);
IPProxy ipProxy = null;
HttpHelper http = new HttpHelper();
//尝试登陆
while (true)
{
try
{
ipProxy = Settings.GetIPProxy();
if (ipProxy == null || string.IsNullOrEmpty(ipProxy.IP))
{
Settings.SimulateCookies = string.Empty;
//return true;
}
HttpItem item = new HttpItem()
{
URL = "https://passport.fang.com/login.api", //URL 必需项
Encoding = null, //编码格式（utf-8,gb2312,gbk） 可选项 默认类会自动识别
//Encoding = Encoding.Default,
Method = "post", //URL 可选项 默认为Get
//Timeout = 100000,//连接超时时间 可选项默认为100000
//ReadWriteTimeout = 30000,//写入Post数据超时时间 可选项默认为30000
//IsToLower = false,//得到的HTML代码是否转成小写 可选项默认转小写
//Cookie = "",//字符串Cookie 可选项
UserAgent = "Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0) like Gecko", //用户的浏览器类型，版本，操作系统 可选项有默认值
Accept = "text/html, application/xhtml+xml, */*", // 可选项有默认值
ContentType = "application/x-www-form-urlencoded", //返回类型 可选项有默认值
Referer = "https://passport.fang.com/", //来源URL 可选项
Postdata = "Uid=luckymn&Pwd=1c523e9b2109407d0857676dfc20af997c14791f495ec8676979628bfef0762ce2679e2f4770d536526bcf00639ec803539f02c54387fbd4a3f159ec5a6185cd46cb139b5c2696c269bce5b7f9c00fb3a9bc58e815773c227b54d4570da0cbee50b47b29c363d398791d3065c0343494aebaa925313e705fd514898e56c2df29&Service=soufun-passport-web&IP=&VCode=&AutoLogin=1",
Allowautoredirect = true,
};
if (ipProxy != null)
{
item.ProxyIp = ipProxy.IP;
}
Console.WriteLine(string.Format("尝试登陆{0}", Settings.curIPProxy != null ? Settings.curIPProxy.IP : string.Empty));
HttpResult result = http.GetHtml(item);
string cookie = string.Empty;
foreach (CookieItem s in HttpCookieHelper.GetCookieList(result.Cookie))
{
//if (s.Key.Contains("24a79_"))
{
cookie += HttpCookieHelper.CookieFormat(s.Key, s.Value);
}
}
if (result.Html.IndexOf("luckymn") > 0)
{
Settings.SimulateCookies = cookie;//设置cookie值
Console.WriteLine("zluckymn模拟登陆成功");
return(true);
}
return(false);
}
catch (WebException ex)
{
IPInvalidProcess(ipProxy);
}
catch (Exception ex)
{
IPInvalidProcess(ipProxy);
}
}
}
/// <summary>
/// 模拟登陆，ip代理可能需要用到
/// </summary>
/// <returns></returns>
public bool SimulateLogin()
{
if (isSpecialUrlMode)
{
return(true);
}
//之前似乎否已经登陆过了
if (!string.IsNullOrEmpty(Settings.LoginAccount))
{
DBChangeQueue.Instance.EnQueue(new StorageData()
{
Document = new BsonDocument().Add(DataTableName + "status", "1"), Name = DataTableNameAccount, Query = Query.EQ("userName", Settings.LoginAccount), Type = StorageType.Update
});
StartDBChangeProcess();
}
var accountBson = AccountQueue.Instance.DeQueue();
if (accountBson == null)
{
Console.WriteLine("账号已用完");
Environment.Exit(0);
return(false);
}
IPProxy ipProxy = null;
HttpHelper http = new HttpHelper();
//尝试登陆
while (true)
{
try
{
ipProxy = Settings.GetIPProxy();
if (ipProxy == null || string.IsNullOrEmpty(ipProxy.IP))
{
Settings.SimulateCookies = string.Empty;
//return true;
}
HttpItem item = new HttpItem()
{
URL = "https://passport.fang.com/login.api", //URL 必需项
Encoding = null, //编码格式（utf-8,gb2312,gbk） 可选项 默认类会自动识别
//Encoding = Encoding.Default,
Method = "post", //URL 可选项 默认为Get
//Timeout = 100000,//连接超时时间 可选项默认为100000
//ReadWriteTimeout = 30000,//写入Post数据超时时间 可选项默认为30000
//IsToLower = false,//得到的HTML代码是否转成小写 可选项默认转小写
//Cookie = "",//字符串Cookie 可选项
UserAgent = "Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0) like Gecko", //用户的浏览器类型，版本，操作系统 可选项有默认值
Accept = "text/html, application/xhtml+xml, */*", // 可选项有默认值
ContentType = "application/x-www-form-urlencoded", //返回类型 可选项有默认值
Referer = "https://passport.fang.com/", //来源URL 可选项
Allowautoredirect = true,
};
if (accountBson != null)
{
item.Postdata = accountBson.Text("postData");
}
if (ipProxy != null)
{
item.ProxyIp = ipProxy.IP;
}
if (Settings.CurWebProxy != null)
{
item.WebProxy = Settings.CurWebProxy;
}
Console.WriteLine(string.Format("尝试登陆{0}", Settings.curIPProxy != null ? Settings.curIPProxy.IP : string.Empty));
HttpResult result = http.GetHtml(item);
if (!result.Html.Contains("Success"))
{
return(false);
}
string cookie = string.Empty;
foreach (CookieItem s in HttpCookieHelper.GetCookieList(result.Cookie))
{
//if (s.Key.Contains("24a79_"))
{
cookie += HttpCookieHelper.CookieFormat(s.Key, s.Value);
}
}
if (accountBson != null)
{
var account = accountBson.Text("userName");
if (result.Html.IndexOf("Success") > 0)
{
Settings.SimulateCookies = cookie;//设置cookie值
Settings.LoginAccount = account;
Console.WriteLine(string.Format("{0}模拟登陆成功", account));
return(true);
}
DBChangeQueue.Instance.EnQueue(new StorageData()
{
Document = new BsonDocument().Add("status", "1"), Name = DataTableNameAccount, Query = Query.EQ("userName", account), Type = StorageType.Update
});
StartDBChangeProcess();
}
else
{
Settings.SimulateCookies = cookie;//设置cookie值
Console.WriteLine("登陆失败");
return(false);
}
return(false);
}
catch (WebException ex)
{
IPInvalidProcess(ipProxy);
}
catch (Exception ex)
{
IPInvalidProcess(ipProxy);
}
}
}
/// <summary>
/// 模拟登陆，ip代理可能需要用到
/// </summary>
/// <returns></returns>
public bool SimulateLogin()
{
//return true;
Settings.SimulateCookies = "yunsuo_session_verify=760b6fed201cabba9dbbc08d6ee95433; yoursessionname1=217703E3DF30F6B367DEC0B3B4EBD13F; yoursessionname0=2ACF87288AD50420E1507A8F20B69186";//设置cookie值
return(true);
if (!string.IsNullOrEmpty(Settings.LoginAccount))
{
return(true);
}
// return SimulateLoginEx();
var userName = string.Empty;
var passWord = string.Empty;
if (AccountQueue.Count() > 0)
{
var _curCookie = AccountQueue.Dequeue();
if (_curCookie != null)
{
Console.WriteLine("提取账号{0}", _curCookie.Text("name"));
userName = _curCookie.Text("name");
passWord = _curCookie.Text("password");
Settings.LoginAccount = userName;
}
else
{
Environment.Exit(0);
}
}
else
{
Environment.Exit(0);
}
// return AliyunAutoLoin(userName, passWord);
IPProxy ipProxy = null;
HttpManager.Instance.InitWebClient(hi, true, 30, 30);
Random rand = new Random(Environment.TickCount);
var nameNormal = userName;
var pwdNormal = passWord;
if (string.IsNullOrEmpty(nameNormal) || string.IsNullOrEmpty(pwdNormal))
{
return(false);
}
var validUrl = "";
var postFormat = "";
bool result = false;
// var postFormat = "geetest_challenge={0}&geetest_validate={1}&geetest_seccode={1}%7Cjordan&requestType=search_enterprise";
var passResult = geetestHelper.PassGeetest(hi, postFormat, validUrl, Settings.SimulateCookies);
result = passResult.Status;
// this.richTextBoxInfo.Document.Blocks.Clear();
//this.webBrowser.Refresh();
if (passResult.Status)
{
hi.Url = "http://www.qichacha.com/user_loginaction";
hi.Refer = "http://www.qichacha.com/user_login";
hi.PostData = string.Format("nameNormal={0}&pwdNormal={1}&geetest_challenge={2}&geetest_validate={3}&geetest_seccode={3}%7Cjordan", nameNormal, pwdNormal, passResult.Challenge, passResult.ValidCode);
var ho = HttpManager.Instance.ProcessRequest(hi);
if (ho.IsOK)
{
if (ho.TxtData.Contains("true"))
{
Settings.SimulateCookies = ho.Cookies;
Console.WriteLine("过验证码模拟登陆成功");
return(true);
}
}
var resultText = geetestHelper.GetLastPoint(hi);
Console.WriteLine(resultText);
}
return(false);
}
public bool SimulateLogin()
{
return(true);
IPProxy ipProxy = null;
if (Settings.IPProxyList == null || Settings.IPProxyList.Where(c => c.Unavaiable == false).Count() <= 0)
{
Environment.Exit(0);
}
HttpHelper http = new HttpHelper();
//尝试登陆
while (true)
{
try
{
ipProxy = Settings.GetIPProxy();
if (ipProxy == null || string.IsNullOrEmpty(ipProxy.IP))
{
Settings.SimulateCookies = string.Empty;
//return true;
}
HttpItem item = new HttpItem()
{
URL = "http://luckymn.cn/QuestionAnswer", //URL 必需项
Encoding = null, //编码格式（utf-8,gb2312,gbk） 可选项 默认类会自动识别
Method = "GET", //URL 可选项 默认为Get
ContentType = "text/html", //返回类型 可选项有默认值
KeepAlive = true,
Timeout = 2000
};
if (ipProxy != null)
{
item.ProxyIp = ipProxy.IP;
}
Console.WriteLine(string.Format("尝试登陆{0}", Settings.curIPProxy != null ? Settings.curIPProxy.IP : string.Empty));
HttpResult result = http.GetHtml(item);
if (result.StatusCode == HttpStatusCode.OK)
{
string cookie = string.Empty;
foreach (CookieItem s in HttpCookieHelper.GetCookieList(result.Cookie))
{
//if (s.Key.Contains("24a79_"))
{
cookie += HttpCookieHelper.CookieFormat(s.Key, s.Value);
}
}
Console.WriteLine("{0}访问成功", item.ProxyIp);
return(true);
}
return(false);
}
catch (WebException ex)
{
IPInvalidProcess(ipProxy);
}
catch (Exception ex)
{
IPInvalidProcess(ipProxy);
}
}
}
/// <summary>
/// ip无效处理
/// </summary>
private void IPInvalidProcess(IPProxy ipproxy)
{
}
/// <summary>
/// 模拟登陆，ip代理可能需要用到
/// </summary>
/// <returns></returns>
public bool SimulateLogin_abort()
{
//if (!string.IsNullOrEmpty(Settings.LoginAccount))
//{
// if (canSimulateLoginEx)
// {
// if (SimulateLoginEx())
// {
// return true;
// }
// }
// else
// {
// return false;
// }
//}
// return SimulateLoginEx();
//Settings.SimulateCookies = "pgv_pvid=2450963536; hide-download-panel=1; _alicdn_sec=576a3ab0af22f4e5ebdbcefe41d61e787594cc18; aliyungf_tc=AQAAAIZesm/zKwsAIkg9O1Mu6IT2uP2O; oldFlag=1; connect.sid=s%3ACds536zVk0sMJohB3xK92r1aqLLzZ2kS.S7xPCWjVBh%2Fv4HTndSuceVBHiz8qzbU4mOW27BoNNlk; hide-index-popup=1; userKey=QXBAdmin-Web2.0_RaVYNd5IpN6lVyiaV9k9vkzHXo5L8gWDaXE0zTdpUUM%3D; userValue=f56373d4-f3bf-74aa-9e60-c2e716af57a7; Hm_lvt_52d64b8d3f6d42a2e416d59635df3f71=1464663566,1464942208,1466478326,1466495053; Hm_lpvt_52d64b8d3f6d42a2e416d59635df3f71=1466579551";//设置cookie值
//return true;
var userName = string.Empty;
var passWord = string.Empty;
if (AccountQueue.Count() > 0)
{
var _curCookie = AccountQueue.Dequeue();
if (_curCookie != null)
{
Console.WriteLine("提取账号{0}", _curCookie.Text("name"));
userName = _curCookie.Text("name");
passWord = _curCookie.Text("password");
Settings.LoginAccount = userName;
}
else
{
Environment.Exit(0);
}
}
else
{
Environment.Exit(0);
}
IPProxy ipProxy = null;
//if (Settings.IPProxyList == null || Settings.IPProxyList.Count() <= 0)
//{
// Environment.Exit(0);
//}
// HttpHelper http = new HttpHelper();
HttpManager.Instance.InitWebClient(hi, true, 30, 30);
Random rand = new Random(Environment.TickCount);
//hi.EnableProxy = true;
//hi.ProxyIP = "127.0.0.1";
//hi.ProxyPort = 8888;
//尝试登陆
while (true)
{
try
{
ipProxy = Settings.GetIPProxy();
if (ipProxy == null || string.IsNullOrEmpty(ipProxy.IP))
{
Settings.SimulateCookies = string.Empty;
//return true;
}
var tempCookie = string.Empty;
// //获取临时验证地址
//HttpItem item_login = new HttpItem()
//{
// URL = "http://www.qixin.com/",//URL 必需项
// Encoding = null,//编码格式（utf-8,gb2312,gbk） 可选项 默认类会自动识别
// Method = "GET",//URL 可选项 默认为Get
// ContentType = "text/html",//返回类型 可选项有默认值
// KeepAlive = true,
//};
//HttpResult result_login = http.GetHtml(item_login);
//if (result_login.StatusCode == HttpStatusCode.OK)
//{
// foreach (CookieItem s in HttpCookieHelper.GetCookieList(result_login.Cookie))
// {
// if (!s.Key.Contains("Path"))
// {
// tempCookie += HttpCookieHelper.CookieFormat(s.Key, s.Value);
// }
// }
//}
//获取临时验证地址
HttpItem item = new HttpItem()
{
URL = "http://120.27.110.11:9600/login_biz/login.oko?uid=01161add5a3c4c55bd9c133baa9effd0", //URL 必需项
Encoding = null, //编码格式（utf-8,gb2312,gbk） 可选项 默认类会自动识别
Method = "GET", //URL 可选项 默认为Get
ContentType = "text/html", //返回类型 可选项有默认值
KeepAlive = true,
Timeout = 9000
};
HttpResult result = http.GetHtml(item);
var needPostUrl = string.Empty;
if (result.StatusCode == HttpStatusCode.OK)
{
needPostUrl = result.Html;
Console.WriteLine("获取打码平台地址:{0}", needPostUrl);
}
hi.Url = string.Format("http://120.27.110.11:9600/login_biz/query_money.oko?uid=01161add5a3c4c55bd9c133baa9effd0");
var ho = HttpManager.Instance.ProcessRequest(hi);
if (ho.IsOK)
{
Console.WriteLine("剩余点数{0}", ho.TxtData);
}
var postDate = string.Empty;
hi.Url = string.Format("http://www.qixin.com/service/gtregister?t={0}", GetTimeLikeJS());
ho = HttpManager.Instance.ProcessRequest(hi);
if (ho.IsOK)
{
var ser = new DataContractJsonSerializer(typeof(GtregisterCls));
var ms = new MemoryStream(Encoding.UTF8.GetBytes(ho.TxtData));
GtregisterCls gtregisterResult = (GtregisterCls)ser.ReadObject(ms);
if (gtregisterResult != null && gtregisterResult.status == 0)
{
postDate = string.Format("data={0}|{1}", gtregisterResult.data.gt, gtregisterResult.data.challenge);
Console.WriteLine("获取打码post验证码:{0}", postDate);
}
}
HttpItem item3 = new HttpItem()
{
UserAgent = "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1) ; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E)",
URL = needPostUrl, //URL 必需项
Encoding = null, //编码格式（utf-8,gb2312,gbk） 可选项 默认类会自动识别
Method = "POST", //URL 可选项 默认为Get
ContentType = "application/x-www-form-urlencoded", //返回类型 可选项有默认值
KeepAlive = true,
Timeout = 90000,
Postdata = postDate
//Postdata = "data=9d80817516218c6af63ab41963087b69|5520e63890a83184ff1bfaa67126575b"
};
HttpResult result3 = http.GetHtml(item3);
var challenge = string.Empty;
var validCode = string.Empty;
if (result3.StatusCode == HttpStatusCode.OK && result3.Html.Contains("success"))
{
string[] lastvcode = result3.Html.Replace("success:", string.Empty).Split(new char[] { '|' });
validCode = lastvcode[0];
challenge = lastvcode[1];
Console.WriteLine("提交打码平台成功{0}|{1}", validCode, challenge);
}
else
{
continue;
}
hi.Url = "http://www.qixin.com/service/gtloginvalidate";
hi.PostData = string.Format("geetest_challenge={0}&geetest_validate={1}&geetest_seccode={1}|jordan", challenge, validCode);
ho = HttpManager.Instance.ProcessRequest(hi);
if (ho.IsOK)
{
if (ho.TxtData.Contains("success"))
{
hi.Url = "http://www.qixin.com/service/login";
hi.Refer = "http://www.qixin.com/login?returnURL=http%3A%2F%2Fwww.qixin.com%2Fcompany%2Fae71e9ad-81f8-4400-88bf-042dd547c93d";
hi.PostData = string.Format("userAcct={0}&userPassword={1}&token={2}%7Cjordan", userName, passWord, validCode);
ho = HttpManager.Instance.ProcessRequest(hi);
if (ho.IsOK)
{
if (ho.TxtData.Contains("成功"))
{
canSimulateLoginEx = true;
Settings.SimulateCookies = ho.Cookies;
Console.WriteLine("过验证码模拟登陆成功");
return(true);
}
}
}
else
{
continue;
}
}
return(false);
}
catch (WebException ex)
{
canSimulateLoginEx = false;
IPInvalidProcess(ipProxy);
}
catch (Exception ex)
{
canSimulateLoginEx = false;
IPInvalidProcess(ipProxy);
}
}
}
/// <summary>
/// 模拟登陆，ip代理可能需要用到
/// </summary>
/// <returns></returns>
public bool SimulateLogin()
{
//if (!string.IsNullOrEmpty(Settings.LoginAccount))
//{
// if (canSimulateLoginEx)
// {
// if (SimulateLoginEx())
// {
// return true;
// }
// }
//}
// return SimulateLoginEx();
var userName = string.Empty;
var passWord = string.Empty;
if (AccountQueue.Count() > 0)
{
var _curCookie = AccountQueue.Dequeue();
if (_curCookie != null)
{
Console.WriteLine("提取账号{0}", _curCookie.Text("name"));
userName = _curCookie.Text("name");
passWord = _curCookie.Text("password");
Settings.LoginAccount = userName;
}
else
{
Environment.Exit(0);
}
}
else
{
Environment.Exit(0);
}
IPProxy ipProxy = null;
HttpManager.Instance.InitWebClient(hi, true, 30, 30);
Random rand = new Random(Environment.TickCount);
//hi.EnableProxy = true;
//hi.ProxyIP = "127.0.0.1";
//hi.ProxyPort = 8888;
//尝试登陆
while (true)
{
try
{
ipProxy = Settings.GetIPProxy();
if (ipProxy == null || string.IsNullOrEmpty(ipProxy.IP))
{
Settings.SimulateCookies = string.Empty;
//return true;
}
var tempCookie = string.Empty;
hi.Url = string.Format("http://www.qixin.com/service/gtregister?t={0}", GetTimeLikeJS());
var postFormat = "geetest_challenge={0}&geetest_validate={1}&geetest_seccode={1}|jordan";
var validUrl = "http://www.qixin.com/service/gtloginvalidate";
var passResult = geetestHelper.PassGeetest(hi, postFormat, validUrl);
if (passResult.Status == true)
{
hi.Url = "http://www.qixin.com/service/login";
hi.Refer = "http://www.qixin.com/login?returnURL=http%3A%2F%2Fwww.qixin.com%2Fcompany%2Fae71e9ad-81f8-4400-88bf-042dd547c93d";
hi.PostData = string.Format("userAcct={0}&userPassword={1}&token={2}%7Cjordan", userName, passWord, passResult.ValidCode);
var ho = HttpManager.Instance.ProcessRequest(hi);
if (ho.IsOK)
{
if (ho.TxtData.Contains("成功"))
{
canSimulateLoginEx = true;
Settings.SimulateCookies = ho.Cookies;
Console.WriteLine("过验证码模拟登陆成功");
return(true);
}
}
}
else
{
continue;
}
return(false);
}
catch (WebException ex)
{
canSimulateLoginEx = false;
IPInvalidProcess(ipProxy);
}
catch (Exception ex)
{
canSimulateLoginEx = false;
IPInvalidProcess(ipProxy);
}
}
}
/// <summary>
///
/// </summary>
/// <param name="proxy"></param>
public void RemoveProxy(IPProxy proxy)
{
IPProxies.ToList().Remove(proxy);
}
/// <summary>
///
/// </summary>
/// <param name="content"></param>
protected override void ParseProxyPage(string content)
{
var doc = new HtmlDocument();
doc.LoadHtml(content);
var document = doc.DocumentNode;
var target = ScraperBox.Helper.GetNodeByAttribute(document, "table", "id", "proxylist");
if (target == null)
{
return;
}
var lines = ScraperBox.Helper.GetNodeCollection(target, "tr");
lines.ToList().ForEach(e =>
{
if (e.Descendants("td").Count() <= 1)
{
return;
}
var cells = e.Descendants("td").ToArray();
var unwanted = cells[0].Descendants("script");
var enumerable = unwanted as HtmlNode[] ?? unwanted.ToArray();
if (enumerable.ToArray().Any())
{
enumerable.ToArray()[0].Remove();
}
var proxy = new IPProxy()
{
ProviderId = GetType().Name.Replace("Cartridge", ""),
//Valid = true
};
// get address's part
try
{
var address = cells[0];
var addressParts = address.InnerText.Split(new[] { ':' }, StringSplitOptions.RemoveEmptyEntries);
var ipAddress = addressParts[1].Replace("port", "").Trim();
var portNo = int.Parse(addressParts[2].Trim());
proxy.IPAddress = ipAddress;
proxy.PortNo = portNo;
}
catch (Exception exception)
{
Console.WriteLine(exception);
//throw;
}
// get anonymity level
proxy.AnonymityLevel = cells[1].InnerText.Trim().Contains("anonymous")
? IPProxyRules.ProxyAnonymityLevelsEnum.Anonymous
: IPProxyRules.ProxyAnonymityLevelsEnum.Elite;
// get last checked time : Apr-27, 16:22
var checkdateTxt = cells[2].InnerText.Replace("Checked:", "").Trim();
var checkdateParts = checkdateTxt.Split(new[] { ',' }, StringSplitOptions.RemoveEmptyEntries);
var checkdateCalPrts = checkdateParts[0].Split(new[] { '-' }, StringSplitOptions.RemoveEmptyEntries);
//"2009 Apr 8 14:40:52,531 <--> yyyy-MM-dd HH:mm:ss,fff
var checkDate = $"{checkdateCalPrts[1]} {checkdateCalPrts[0]} {DateTime.Now.Year}";
var checkDatePrsd = DateTime.ParseExact(checkDate, "dd MMM yyyy", System.Globalization.CultureInfo.InvariantCulture);
proxy.LastChecked = checkDatePrsd.Add(TimeSpan.Parse(checkdateParts[1]));
//todo:
//proxy.LastValidationCheck = DateTime.Parse(checkdate);
// get the country
var countryPartial = cells[3].InnerText
.Replace("Country:", "")
.Trim().Replace(" ", "_")
.ToLower();
if (!string.IsNullOrEmpty(countryPartial))
{
countryPartial = countryPartial.Split(new[] { ',' }, StringSplitOptions.RemoveEmptyEntries)[0];
proxy.Country = ScraperBox.Helper.FindProxyCountryFromPartial(countryPartial);
}
proxy.City = cells[4].InnerText.Replace("City:", "").Replace(" ", "").Trim();
proxy.ISP = cells[5].InnerText.Replace("ISP:", "");
//if (!ProxyTestHelper.CanPing(string.Format("{0}://{1}:{2}", proxy.Protocol == ProxyProtocolEnum.HTTP ? "http" : "https", proxy.HostIP, proxy.PortNo)))
//if (!ProxyTestHelper.ProxyIsGood(proxy.HostIP, proxy.PortNo)) return;
RegisterProxy(proxy);
});
base.ParseProxyPage(content);
}
public FreeIPProxyParsedEventArgs(int pageNo, string targetPageUrl, IPProxy proxy)
{
TargetPageUrl = targetPageUrl;
IPProxy = proxy;
PageNo = pageNo;
}
/// <summary>
///
/// </summary>
/// <param name="content"></param>
protected override void ParseProxyPage(string content)
{
var doc = new HtmlDocument();
doc.LoadHtml(content);
var document = doc.DocumentNode;
var target = HtmlUtil.GetNodeByAttribute(document, "table", "class", "proxytbl");
if (target == null)
{
return;
}
var lines = HtmlUtil.GetNodeCollection(target, "tr");
lines.ToList().ForEach(e =>
{
if (e.Descendants("td").Count() <= 1)
{
return;
}
var proxy = new IPProxy()
{
ProviderId = GetType().Name.Replace("Cartridge", ""),
};
var cells = e.Descendants("td").ToArray();
var scriptPart = cells[1].Descendants("script");
var htmlNodes = scriptPart as HtmlNode[] ?? scriptPart.ToArray();
if (htmlNodes.Any())
{
htmlNodes.ToArray()[0].Remove();
}
//ip
proxy.IPAddress = cells[0].InnerText.Trim();
//port
proxy.PortNo = int.Parse(HtmlUtil.Resolve(cells[1].InnerText.Trim()).Replace("\r\n", "").Trim());
// country
var country = HtmlUtil.Resolve(cells[2].InnerText.Trim());
var countryPrts = country.Split(new[] { ',' }, StringSplitOptions.RemoveEmptyEntries);
proxy.Country = Helper.FindProxyCountryFromPartial(countryPrts[0].Replace(" ", "_"));
// anon
proxy.AnonymityLevel = cells[3].InnerText.Trim().Replace("\r\n", "").Trim().Contains("anonymous")
? ProxyAnonymityLevelsEnum.Anonymous
: ProxyAnonymityLevelsEnum.Elite;
//protocol
var https = HtmlUtil.Resolve(cells[4].Attributes["class"].Value);
proxy.Protocol = https.ToLower().Contains("https")
? ProxyProtocolsEnum.HTTPS
: ProxyProtocolsEnum.HTTP;
//last check
var lastChecked = (new DateTime(DateTime.Now.Year, DateTime.Now.Month, DateTime.Now.Day));
proxy.LastChecked = lastChecked.Add(TimeSpan.Parse(HtmlUtil.Resolve(cells[5].InnerText.Trim())));
RegisterProxy(proxy);
});
base.ParseProxyPage(content);
}
public IPProxyCheckedEventArgs(bool isValid, IPProxy proxy)
{
IsValid = isValid;
IPProxy = proxy;
}
/// <summary>
/// 模拟登陆，ip代理可能需要用到
/// </summary>
/// <returns></returns>
public bool SimulateLogin()
{
//Settings.SimulateCookies = "pgv_pvid=1513639250; aliyungf_tc=AQAAADPRHnGrvwQAIkg9OwOqtkYJbU4N; oldFlag=1; CNZZDATA1259577625=112366950-1466409958-%7C1466415358; hide-index-popup=1; hide-download-panel=1; _alicdn_sec=576ba5f9a986fb4802dacf51bc99b1e76724f58e; connect.sid=s%3AeYWXycPKai63BYTmB9d6h-0IM_R2kp6n.EUgfW0AmJ6GB%2F0TamTi4tT53QK4OR4yQtU1I3Ba8Ryo; userKey=QXBAdmin-Web2.0_N3iUdNobAoys4M395Pk5v%2F6Zxcwjt1tiCqeSf3X3ZnI%3D; userValue=bea26f0d-e414-168a-0fe2-b8eb4278ab07; Hm_lvt_52d64b8d3f6d42a2e416d59635df3f71=1464663982,1464775028,1464776749,1465799273; Hm_lpvt_52d64b8d3f6d42a2e416d59635df3f71=1466672591";//设置cookie值
//return true;
if (!string.IsNullOrEmpty(Settings.LoginAccount))
{
if (canSimulateLoginEx)
{
if (SimulateLoginEx())
{
return(true);
}
}
else
{
return(false);
}
}
// return SimulateLoginEx();
var userName = string.Empty;
var passWord = string.Empty;
if (AccountQueue.Count() > 0)
{
var _curCookie = AccountQueue.Dequeue();
if (_curCookie != null)
{
Console.WriteLine("提取账号{0}", _curCookie.Text("name"));
userName = _curCookie.Text("name");
passWord = _curCookie.Text("password");
Settings.LoginAccount = userName;
}
else
{
Environment.Exit(0);
}
}
else
{
Environment.Exit(0);
}
IPProxy ipProxy = null;
HttpManager.Instance.InitWebClient(hi, true, 30, 30);
Random rand = new Random(Environment.TickCount);
//尝试登陆
while (true)
{
try
{
ipProxy = Settings.GetIPProxy();
if (ipProxy == null || string.IsNullOrEmpty(ipProxy.IP))
{
Settings.SimulateCookies = string.Empty;
//return true;
}
var tempCookie = string.Empty;
hi.Url = string.Format("http://www.qixin.com/service/gtregister?t={0}", GetTimeLikeJS());
var postFormat = "geetest_challenge={0}&geetest_validate={1}&geetest_seccode={1}|jordan";
var validUrl = "http://www.qixin.com/service/gtloginvalidate";
var passResult = geetestHelper.PassGeetest(hi, postFormat, validUrl);
if (passResult.Status == true)
{
hi.Url = "http://www.qixin.com/service/login";
hi.Refer = "http://www.qixin.com/login?returnURL=http%3A%2F%2Fwww.qixin.com%2Fcompany%2Fae71e9ad-81f8-4400-88bf-042dd547c93d";
hi.PostData = string.Format("userAcct={0}&userPassword={1}&token={2}%7Cjordan", userName, passWord, passResult.ValidCode);
var ho = HttpManager.Instance.ProcessRequest(hi);
if (ho.IsOK)
{
if (ho.TxtData.Contains("成功"))
{
canSimulateLoginEx = true;
Settings.SimulateCookies = ho.Cookies;
Console.WriteLine("过验证码模拟登陆成功");
return(true);
}
}
}
else
{
continue;
}
return(false);
}
catch (WebException ex)
{
canSimulateLoginEx = false;
IPInvalidProcess(ipProxy);
}
catch (Exception ex)
{
canSimulateLoginEx = false;
IPInvalidProcess(ipProxy);
}
}
}
