public static JArray CrawlerSingleQuestion(string questionId)
{
try
{
#if DEBUG
var a =
HttpWebResponseProxyMayi.ExecuteCreateGetHttpResponseProxy(
$"https://www.zujuan.com/question/detail-{questionId}.shtml", 6000, null);
//var a = HttpClientHolder.Proxy_GetRequest2();
#else
var a = HttpClientHolder.Proxy_GetRequestAbyyun($"https://www.zujuan.com/question/detail-{questionId}.shtml");
#endif
//
if (a.IndexOf("试题已经被删除") >= 0)
{
Console.WriteLine($"{questionId} has delete!!!");
using (var db = new CrawlerEntities())
{
var id = questionId.NullToInt();
var entity = db.Question.FirstOrDefault(t => t.Question_Id == id);
if (entity != null)
{
entity.IsGrabAns = true;
entity.IsRemoteDelete = true;
db.SaveChanges();
}
}
return(null);
}
if (a.IndexOf("限制访问试题") >= 0)
{
using (var db = new CrawlerEntities())
{
var id = questionId.NullToInt();
var entity = db.Question.FirstOrDefault(t => t.Question_Id == id);
if (entity != null)
{
entity.IsGrabAns = false;
db.SaveChanges();
}
}
return(null);
}
Console.WriteLine($"start crawler https://www.zujuan.com/question/detail-{questionId}.shtml");
//例如我想提取记录中的NAME值
string value = GetValue(a, "var MockDataTestPaper =", "OT2.renderQList").TrimEnd(new char[] { ';' });
value = value.Trim().TrimEnd(new char[] { ';' }).Trim();
JArray jObject = JArray.Parse(value);
if (!string.IsNullOrWhiteSpace(jObject[0]["questions"][0]["list"].NullToString()))
{
if (string.IsNullOrWhiteSpace(jObject[0]["questions"][0]["list"][0]["answer"].NullToString()))
{
Console.WriteLine("明细题答案抓取错误");
Debug.WriteLine("明细题答案抓取错误");
throw new Exception("明细题答案抓取错误");
}
}
//; UpdateProxGrabyime(proxy.Id);
Console.WriteLine($"aleady get {questionId} return value");
//更新代理时间
return(JArray.Parse(value));
}
catch (IOException io)
{
using (var db = new CrawlerEntities())
{
var id = questionId.NullToInt();
var entity = db.Question.FirstOrDefault(t => t.Question_Id == id);
if (entity != null)
{
entity.IsRemoteDelete = false;
entity.IsGrabAns = false;
db.SaveChanges();
}
}
}
catch (JsonReaderException je)
{
using (var db = new CrawlerEntities())
{
var id = questionId.NullToInt();
var entity = db.Question.FirstOrDefault(t => t.Question_Id == id);
if (entity != null)
{
entity.IsRemoteDelete = false;
entity.IsGrabAns = false;
db.SaveChanges();
}
}
}
catch (WebException we)
{
using (var db = new CrawlerEntities())
{
var id = questionId.NullToInt();
var entity = db.Question.FirstOrDefault(t => t.Question_Id == id);
if (entity != null)
{
entity.IsRemoteDelete = false;
entity.IsGrabAns = false;
db.SaveChanges();
}
}
}
catch (Exception e)
{
using (var db = new CrawlerEntities())
{
var id = questionId.NullToInt();
var entity = db.Question.FirstOrDefault(t => t.Question_Id == id);
if (entity != null)
{
entity.IsRemoteDelete = false;
entity.IsGrabAns = false;
db.SaveChanges();
}
}
}
//UpdateQuestionGrabStatus(questionId.NullToInt());
return(null);
}
public static List <string> GetProxyListFromBy()
{
var res = HttpClientHolder.GetRequest("http://www.flnsu.com/ip.php?key=9172343651&tqsl=500");
return(res.Split(Environment.NewLine.ToCharArray(), StringSplitOptions.RemoveEmptyEntries).ToList());
}
public static JArray CrawlerSingleQuestion(string questionId, List <string> proxys)
{
var proxy = proxys[new Random().Next(0, proxys.Count - 1)];
try
{
var a = HttpClientHolder.Proxy_GetRequest($"https://www.zujuan.com/question/detail-{questionId}.shtml",
proxy);
if (a.IndexOf("试题已经被删除") >= 0)
{
Console.WriteLine($"{questionId} has delete!!!");
UpdateQuestionRemoteStatus(questionId.ToInt());
return(null);
}
if (a.IndexOf("限制访问试题") >= 0)
{
using (var db = new CrawlerEntities())
{
var id = questionId.NullToInt();
var entity = db.QuestionAll.FirstOrDefault(t => t.QuestionId == id);
if (entity != null)
{
entity.IsGrabAns = false;
db.SaveChanges();
}
}
return(null);
}
Console.WriteLine($"start crawler https://www.zujuan.com/question/detail-{questionId}.shtml {proxy}");
//例如我想提取记录中的NAME值
string value = GetValue(a, "var MockDataTestPaper =", "OT2.renderQList").TrimEnd(new char[] { ';' });
value = value.Trim().TrimEnd(new char[] { ';' }).Trim();
//; UpdateProxGrabyime(proxy.Id);
Console.WriteLine($"aleady get {questionId} return value");
//更新代理时间
return(JArray.Parse(value));
}
catch (IOException io)
{
if (!UnusefulProxy.Any(t => t.Equals(proxy)))
{
UnusefulProxy.Add(proxy);
}
}
catch (JsonReaderException je)
{
UpdateQuestionRemoteStatus(questionId.ToInt());
}
catch (WebException we)
{
using (var db = new CrawlerEntities())
{
var id = questionId.NullToInt();
var entity = db.QuestionAll.FirstOrDefault(t => t.QuestionId == id);
if (entity != null)
{
entity.IsGrabAns = false;
db.SaveChanges();
}
}
//if (we.Status == WebExceptionStatus.ConnectFailure || we.Status == WebExceptionStatus.ProtocolError)
//{
if (!UnusefulProxy.Any(t => t.Equals(proxy)))
{
UnusefulProxy.Add(proxy);
}
//SetProxyDisable(proxy.Id);
//}
}
catch (Exception e)
{
using (var db = new CrawlerEntities())
{
var id = questionId.NullToInt();
var entity = db.QuestionAll.FirstOrDefault(t => t.QuestionId == id);
if (entity != null)
{
entity.IsGrabAns = false;
db.SaveChanges();
}
}
}
//UpdateQuestionGrabStatus(questionId.NullToInt());
return(null);
}