public static JArray CrawlerSingleQuestion(int questionId, List <string> proxys)
{
var proxy = proxys[new Random().Next(0, proxys.Count - 1)];
try
{
var a = HttpClientHolder.Proxy_GetRequest($"https://www.zujuan.com/question/detail-{questionId}.shtml",
proxy);
if (a.IndexOf("试题已经被删除") >= 0)
{
// AddDeleteQuestion(questionId);
}
if (a.IndexOf("限制访问试题") >= 0)
{
throw new Exception("test");
}
Console.WriteLine($"start crawler https://www.zujuan.com/question/detail-{questionId}.shtml {proxy}");
//例如我想提取记录中的NAME值
string value = GrabAnswer.GetValue(a, "var MockDataTestPaper =", "OT2.renderQList").TrimEnd(new char[] { ';' });
value = value.Trim().TrimEnd(new char[] { ';' }).Trim();
//; UpdateProxGrabyime(proxy.Id);
//更新代理时间
return(JArray.Parse(value));
}
catch (IOException io)
{
if (!UnusefulProxy.Any(t => t.Equals(proxy)))
{
UnusefulProxy.Add(proxy);
}
}
catch (JsonReaderException je)
{
//AddDeleteQuestion(questionId);
}
catch (WebException we)
{
//if (we.Status == WebExceptionStatus.ConnectFailure || we.Status == WebExceptionStatus.ProtocolError)
//{
if (!UnusefulProxy.Any(t => t.Equals(proxy)))
{
UnusefulProxy.Add(proxy);
}
//SetProxyDisable(proxy.Id);
//}
}
return(null);
}
public static JArray CrawlerSingleQuestion(string questionId, List <string> proxys)
{
var proxy = proxys[new Random().Next(0, proxys.Count - 1)];
try
{
var a = HttpClientHolder.Proxy_GetRequest($"https://www.zujuan.com/question/detail-{questionId}.shtml",
proxy);
if (a.IndexOf("试题已经被删除") >= 0)
{
Console.WriteLine($"{questionId} has delete!!!");
UpdateQuestionRemoteStatus(questionId.ToInt());
return(null);
}
if (a.IndexOf("限制访问试题") >= 0)
{
using (var db = new CrawlerEntities())
{
var id = questionId.NullToInt();
var entity = db.QuestionAll.FirstOrDefault(t => t.QuestionId == id);
if (entity != null)
{
entity.IsGrabAns = false;
db.SaveChanges();
}
}
return(null);
}
Console.WriteLine($"start crawler https://www.zujuan.com/question/detail-{questionId}.shtml {proxy}");
//例如我想提取记录中的NAME值
string value = GetValue(a, "var MockDataTestPaper =", "OT2.renderQList").TrimEnd(new char[] { ';' });
value = value.Trim().TrimEnd(new char[] { ';' }).Trim();
//; UpdateProxGrabyime(proxy.Id);
Console.WriteLine($"aleady get {questionId} return value");
//更新代理时间
return(JArray.Parse(value));
}
catch (IOException io)
{
if (!UnusefulProxy.Any(t => t.Equals(proxy)))
{
UnusefulProxy.Add(proxy);
}
}
catch (JsonReaderException je)
{
UpdateQuestionRemoteStatus(questionId.ToInt());
}
catch (WebException we)
{
using (var db = new CrawlerEntities())
{
var id = questionId.NullToInt();
var entity = db.QuestionAll.FirstOrDefault(t => t.QuestionId == id);
if (entity != null)
{
entity.IsGrabAns = false;
db.SaveChanges();
}
}
//if (we.Status == WebExceptionStatus.ConnectFailure || we.Status == WebExceptionStatus.ProtocolError)
//{
if (!UnusefulProxy.Any(t => t.Equals(proxy)))
{
UnusefulProxy.Add(proxy);
}
//SetProxyDisable(proxy.Id);
//}
}
catch (Exception e)
{
using (var db = new CrawlerEntities())
{
var id = questionId.NullToInt();
var entity = db.QuestionAll.FirstOrDefault(t => t.QuestionId == id);
if (entity != null)
{
entity.IsGrabAns = false;
db.SaveChanges();
}
}
}
//UpdateQuestionGrabStatus(questionId.NullToInt());
return(null);
}