public static void CrawlerAnswer(int qid) { var proxys = GrabAnswer.GetProxyListFromCache(); //var answer = new QuestionParser().ParseAnswer(item.QuestionId.ToString()); var answerHttpClient = CrawlerSingleQuestion(qid, proxys); if (answerHttpClient == null || string.IsNullOrWhiteSpace(answerHttpClient.ToString())) { return; } var xd = answerHttpClient[0]["questions"][0]["xd"].ToString(); var chid = answerHttpClient[0]["questions"][0]["chid"].ToString(); var apiUrl = $"https://www.zujuan.com/question/list?question_id={qid}&xd={xd}&chid={chid}"; var apiJson = HttpClientHolder.Execute(apiUrl); using (var db = new CrawlerEntities()) { var entity = new QuestionAll(); entity.QuestionId = qid; entity.IsDelete = false; entity.AnswerJson = answerHttpClient.ToString(); entity.ApiJson = apiJson; entity.CrawlerUrl = $"https://www.zujuan.com/question/detail-{qid}.shtml"; entity.CrawlerApiUrl = apiUrl; entity.child = chid.NullToInt(); entity.xd = xd.NullToInt(); db.QuestionAll.Add(entity); db.SaveChanges(); } }
public static JArray CrawlerSingleQuestion(int questionId, List <string> proxys) { var proxy = proxys[new Random().Next(0, proxys.Count - 1)]; try { var a = HttpClientHolder.Proxy_GetRequest($"https://www.zujuan.com/question/detail-{questionId}.shtml", proxy); if (a.IndexOf("试题已经被删除") >= 0) { // AddDeleteQuestion(questionId); } if (a.IndexOf("限制访问试题") >= 0) { throw new Exception("test"); } Console.WriteLine($"start crawler https://www.zujuan.com/question/detail-{questionId}.shtml {proxy}"); //例如我想提取记录中的NAME值 string value = GrabAnswer.GetValue(a, "var MockDataTestPaper =", "OT2.renderQList").TrimEnd(new char[] { ';' }); value = value.Trim().TrimEnd(new char[] { ';' }).Trim(); //; UpdateProxGrabyime(proxy.Id); //更新代理时间 return(JArray.Parse(value)); } catch (IOException io) { if (!UnusefulProxy.Any(t => t.Equals(proxy))) { UnusefulProxy.Add(proxy); } } catch (JsonReaderException je) { //AddDeleteQuestion(questionId); } catch (WebException we) { //if (we.Status == WebExceptionStatus.ConnectFailure || we.Status == WebExceptionStatus.ProtocolError) //{ if (!UnusefulProxy.Any(t => t.Equals(proxy))) { UnusefulProxy.Add(proxy); } //SetProxyDisable(proxy.Id); //} } return(null); }