public static JArray CrawlerSingleQuestion(string questionId) { try { #if DEBUG var a = HttpWebResponseProxyMayi.ExecuteCreateGetHttpResponseProxy( $"https://www.zujuan.com/question/detail-{questionId}.shtml", 6000, null); //var a = HttpClientHolder.Proxy_GetRequest2(); #else var a = HttpClientHolder.Proxy_GetRequestAbyyun($"https://www.zujuan.com/question/detail-{questionId}.shtml"); #endif // if (a.IndexOf("试题已经被删除") >= 0) { Console.WriteLine($"{questionId} has delete!!!"); using (var db = new CrawlerEntities()) { var id = questionId.NullToInt(); var entity = db.Question.FirstOrDefault(t => t.Question_Id == id); if (entity != null) { entity.IsGrabAns = true; entity.IsRemoteDelete = true; db.SaveChanges(); } } return(null); } if (a.IndexOf("限制访问试题") >= 0) { using (var db = new CrawlerEntities()) { var id = questionId.NullToInt(); var entity = db.Question.FirstOrDefault(t => t.Question_Id == id); if (entity != null) { entity.IsGrabAns = false; db.SaveChanges(); } } return(null); } Console.WriteLine($"start crawler https://www.zujuan.com/question/detail-{questionId}.shtml"); //例如我想提取记录中的NAME值 string value = GetValue(a, "var MockDataTestPaper =", "OT2.renderQList").TrimEnd(new char[] { ';' }); value = value.Trim().TrimEnd(new char[] { ';' }).Trim(); JArray jObject = JArray.Parse(value); if (!string.IsNullOrWhiteSpace(jObject[0]["questions"][0]["list"].NullToString())) { if (string.IsNullOrWhiteSpace(jObject[0]["questions"][0]["list"][0]["answer"].NullToString())) { Console.WriteLine("明细题答案抓取错误"); Debug.WriteLine("明细题答案抓取错误"); throw new Exception("明细题答案抓取错误"); } } //; UpdateProxGrabyime(proxy.Id); Console.WriteLine($"aleady get {questionId} return value"); //更新代理时间 return(JArray.Parse(value)); } catch (IOException io) { using (var db = new CrawlerEntities()) { var id = questionId.NullToInt(); var entity = db.Question.FirstOrDefault(t => t.Question_Id == id); if (entity != null) { entity.IsRemoteDelete = false; entity.IsGrabAns = false; db.SaveChanges(); } } } catch (JsonReaderException je) { using (var db = new CrawlerEntities()) { var id = questionId.NullToInt(); var entity = db.Question.FirstOrDefault(t => t.Question_Id == id); if (entity != null) { entity.IsRemoteDelete = false; entity.IsGrabAns = false; db.SaveChanges(); } } } catch (WebException we) { using (var db = new CrawlerEntities()) { var id = questionId.NullToInt(); var entity = db.Question.FirstOrDefault(t => t.Question_Id == id); if (entity != null) { entity.IsRemoteDelete = false; entity.IsGrabAns = false; db.SaveChanges(); } } } catch (Exception e) { using (var db = new CrawlerEntities()) { var id = questionId.NullToInt(); var entity = db.Question.FirstOrDefault(t => t.Question_Id == id); if (entity != null) { entity.IsRemoteDelete = false; entity.IsGrabAns = false; db.SaveChanges(); } } } //UpdateQuestionGrabStatus(questionId.NullToInt()); return(null); }
public static JArray CrawlerSingleQuestion(string questionId, List <string> proxys) { var proxy = proxys[new Random().Next(0, proxys.Count - 1)]; try { var a = HttpClientHolder.Proxy_GetRequestAbyyun($"https://www.zujuan.com/question/detail-{questionId}.shtml"); if (a.IndexOf("试题已经被删除") >= 0) { Console.WriteLine($"{questionId} has delete!!!"); UpdateQuestionRemoteStatus(questionId.ToInt()); return(null); } if (a.IndexOf("限制访问试题") >= 0) { using (var db = new CrawlerEntities()) { var id = questionId.NullToInt(); var entity = db.QuestionAll.FirstOrDefault(t => t.QuestionId == id); if (entity != null) { entity.IsGrabAns = false; db.SaveChanges(); } } return(null); } Console.WriteLine($"start crawler https://www.zujuan.com/question/detail-{questionId}.shtml {proxy}"); //例如我想提取记录中的NAME值 string value = GetValue(a, "var MockDataTestPaper =", "OT2.renderQList").TrimEnd(new char[] { ';' }); value = value.Trim().TrimEnd(new char[] { ';' }).Trim(); //; UpdateProxGrabyime(proxy.Id); Console.WriteLine($"aleady get {questionId} return value"); //更新代理时间 return(JArray.Parse(value)); } catch (IOException io) { if (!UnusefulProxy.Any(t => t.Equals(proxy))) { UnusefulProxy.Add(proxy); } } catch (JsonReaderException je) { UpdateQuestionRemoteStatus(questionId.ToInt()); } catch (WebException we) { using (var db = new CrawlerEntities()) { var id = questionId.NullToInt(); var entity = db.QuestionAll.FirstOrDefault(t => t.QuestionId == id); if (entity != null) { entity.IsGrabAns = false; db.SaveChanges(); } } //if (we.Status == WebExceptionStatus.ConnectFailure || we.Status == WebExceptionStatus.ProtocolError) //{ if (!UnusefulProxy.Any(t => t.Equals(proxy))) { UnusefulProxy.Add(proxy); } //SetProxyDisable(proxy.Id); //} } catch (Exception e) { using (var db = new CrawlerEntities()) { var id = questionId.NullToInt(); var entity = db.QuestionAll.FirstOrDefault(t => t.QuestionId == id); if (entity != null) { entity.IsGrabAns = false; db.SaveChanges(); } } } //UpdateQuestionGrabStatus(questionId.NullToInt()); return(null); }