public bool QueryResumeIsExists(string resumeNumber, DateTime?refreshTime = null) { using (var db = new MangningXssDBEntities()) { var exists = db.ZhaopinWatchedResume.AsNoTracking().Any(f => f.ResumeNumber == resumeNumber); if (exists) { return(true); } var resume = db.ZhaopinResume.FirstOrDefault(f => f.RandomNumber == resumeNumber); if (resume != null) { resume.RefreshTime = refreshTime; db.SaveChanges(); return(true); } } using (var db = new BadoucaiAliyunDBEntities()) { return(db.CoreResumeReferenceMapping.AsNoTracking().Any(a => a.Key == "ResumeNumber" && a.Source == "ZHAOPIN" && a.Value.StartsWith(resumeNumber))); } }
/// <summary> /// 获取验证码 /// </summary> /// <returns></returns> public ZhaopinCheckCode GetCheckCode() { if (waitCount == 0) { return(null); } using (var db = new MangningXssDBEntities()) { var checkCode = db.ZhaopinCheckCode.FirstOrDefault(f => f.Status == 0); if (checkCode == null) { return(null); } checkCode.Status = 1; db.SaveChanges(); dictionary.Add(checkCode.Id, DateTime.Now.AddMinutes(1)); waitCount--; return(checkCode); } }
public void SetSearchStatus(int id, short status, int quantity, int lastWatchPage = -1) { using (var db = new MangningXssDBEntities()) { var condition = db.ZhaopinSearchCondition.FirstOrDefault(f => f.Id == id); if (condition == null) { return; } condition.Status = status; condition.Quantity = quantity; condition.LastSearchDate = DateTime.UtcNow; if (lastWatchPage > 0) { condition.LastWatchPage = lastWatchPage; } db.SaveChanges(); } }
/// <summary> /// 验证结果 /// </summary> /// <param name="checkCodeId"></param> /// <param name="status">1.成功 2.失败</param> /// <param name="cleaningId"></param> /// <returns></returns> public void LoginCheckResult(int checkCodeId, short status, short cleaningId) { if (status == 1) { using (var db = new MangningXssDBEntities()) { var checkCode = db.ZhaopinCheckCode.FirstOrDefault(f => f.Id == checkCodeId); var date = DateTime.Now; if (checkCode == null) { return; } dictionary.Remove(checkCode.Id); checkCode.Status = 2; checkCode.CompleteTime = date; db.SaveChanges(); } } if (status == 2) { if (!dictionary.ContainsKey(checkCodeId)) { return; } dictionary[checkCodeId] = DateTime.Now.AddMinutes(1); } }
/// <summary> /// 启动验证计时器 /// </summary> /// <param name="checkCodeId"></param> public void StartTimer() { while (true) { Thread.Sleep(1000); var list = dictionary.Where(w => DateTime.Now.Subtract(w.Value).Milliseconds > 0).ToList(); using (var db = new MangningXssDBEntities()) { foreach (var item in list) { var checkCode = db.ZhaopinCheckCode.FirstOrDefault(f => f.Id == item.Key); if (checkCode == null) { return; } checkCode.Status = 0; dictionary.Remove(item.Key); db.SaveChanges(); } } } }
/// <summary> /// 启动验证计时器 /// </summary> private void StartTimer() { while (true) { Thread.Sleep(1000); if (endTime == null) { this.RunInMainthread(() => { this.lbl_timer.Text = "倒计时:60s"; }); return; } var seconds = endTime.Value.Subtract(DateTime.Now).Seconds; this.RunInMainthread(() => { this.lbl_timer.Text = $"倒计时:{seconds}s"; }); if (seconds <= 0) { break; } } using (var db = new MangningXssDBEntities()) { var checkCode = db.ZhaopinCheckCode.FirstOrDefault(f => f.HandleUser == handleUser && f.Status != 2); if (checkCode != null) { checkCode.HandleUser = null; checkCode.Status = 0; db.SaveChanges(); } } this.RunInMainthread(() => { this.btn_Checking.Enabled = false; this.btn_GetCheckCode.Enabled = true; this.btn_ReferenceCheckCode.Enabled = false; this.pic_Body.Image = null; this.pic_Header.Image = null; this.lbl_Tip.Text = $"提示:{DateTime.Now:HH:mm:ss} 输入超时,请重新获取验证码!"; }); }
/// <summary> /// 关闭清洗程序 /// </summary> /// <param name="account"></param> public void CleaningEnd(string account) { using (var db = new MangningXssDBEntities()) { var cleaningProcedure = db.ZhaopinCleaningProcedure.FirstOrDefault(f => f.Account == account); if (cleaningProcedure != null) { cleaningProcedure.IsOnline = false; db.SaveChanges(); } } }
/// <summary> /// 加载多迪商机 /// </summary> private static void LoadDodiResume() { using (var db = new MangningXssDBEntities()) { var dodiBusinessList = db.DodiBusiness.Where(w => w.Status == 1).ToList(); foreach (var item in dodiBusinessList) { item.Status = 0; } db.SaveChanges(); } while (true) { if (businessIdQueue.Count > 128) { Thread.Sleep(1000); continue; } try { using (var db = new MangningXssDBEntities()) { var dodiBusinessList = db.DodiBusiness.Where(w => w.Status == 0 && w.Sources.Contains("智联")).Take(1024).ToList(); foreach (var item in dodiBusinessList) { item.Status = 1; } db.SaveChanges(); foreach (var item in dodiBusinessList) { businessIdQueue.Enqueue(item.Id); } } } catch (Exception ex) { Trace.TraceError(ex.ToString()); } } }
public void RemoveCookie(string account, int todayWatch) { using (var db = new MangningXssDBEntities()) { var pro = db.ZhaopinCleaningProcedure.FirstOrDefault(f => f.Account == account); if (pro != null) { pro.Cookie = ""; pro.TodayWatch = todayWatch; pro.IsOnline = false; db.SaveChanges(); } } }
/// <summary> /// 插入验证码 /// </summary> /// <param name="checkCode"></param> public int InserCheckCode(ZhaopinCheckCode checkCode) { using (var db = new MangningXssDBEntities()) { var checkModel = db.ZhaopinCheckCode.FirstOrDefault(a => a.Account == checkCode.Account && (a.Status == 0 || a.Status == 1)); if (checkModel != null) { return(checkModel.Id); } checkCode = db.ZhaopinCheckCode.Add(checkCode); db.SaveChanges(); return(checkCode.Id); } }
/// <summary> /// 启动清洗程序 /// </summary> /// <param name="model"></param> public void CleaningStart(ZhaopinCleaningProcedure model) { using (var db = new MangningXssDBEntities()) { var cleaningProcedure = db.ZhaopinCleaningProcedure.FirstOrDefault(f => f.Id == model.Id); if (cleaningProcedure == null) { db.ZhaopinCleaningProcedure.Add(model); } else { cleaningProcedure.StartTime = DateTime.UtcNow; cleaningProcedure.IsOnline = true; } db.SaveChanges(); } }
/// <summary> /// 验证结果 /// </summary> /// <param name="checkCodeId"></param> /// <param name="status">1.成功 2.失败</param> /// <param name="account"></param> /// <param name="handleUser"></param> /// <returns></returns> public string[] CheckResult(int checkCodeId, short status, string account, string handleUser) { if (status == 1) { using (var db = new MangningXssDBEntities()) { var user = account.Substring(0, account.IndexOf("_", StringComparison.Ordinal)); var checkCodes = db.ZhaopinCheckCode.Where(w => w.Account.StartsWith(user) && (w.Status == 0 || w.Status == 1)).ToList(); var date = DateTime.Now; foreach (var checkCode in checkCodes) { dictionary.Remove(checkCode.Id); checkCode.Status = 2; checkCode.HandleUser = handleUser; checkCode.CompleteTime = date; } db.SaveChanges(); return(checkCodes.Select(s => s.Account).ToArray()); } } if (status == 2) { if (!dictionary.ContainsKey(checkCodeId)) { return(null); } dictionary[checkCodeId] = DateTime.Now.AddMinutes(1); } return(null); }
/// <summary> /// 重置状态 /// </summary> public void ResetStatus() { using (var db = new MangningXssDBEntities()) { var list = db.ZhaopinCheckCode.Where(a => a.Status == 1).ToList(); foreach (var item in list) { item.Status = 0; } var procedures = db.ZhaopinCleaningProcedure.Where(a => a.IsOnline).ToList(); foreach (var item in procedures) { item.IsOnline = false; } db.SaveChanges(); } }
private void btn_GetCheckCode_Click(object sender, EventArgs e) { if (isGetCheckCode) { return; } isGetCheckCode = true; using (var db = new MangningXssDBEntities()) { var checkCode = db.ZhaopinCheckCode.FirstOrDefault(f => f.HandleUser == handleUser && (f.Status == 0 || f.Status == 1)); if (checkCode == null) { db.Database.ExecuteSqlCommand("UPDATE XSS_Zhaopin_CheckCode SET HandleUser = @handleUser WHERE Status = 0 AND HandleUser IS NULL LIMIT 1", new MySqlParameter("@handleUser", handleUser)); checkCode = db.ZhaopinCheckCode.FirstOrDefault(f => f.HandleUser == handleUser && (f.Status == 0 || f.Status == 1)); } if (checkCode == null) { this.lbl_Tip.Text = $"提示:{DateTime.Now:HH:mm:ss} 无验证码待输入!"; isGetCheckCode = false; return; } checkCode.Status = 1; db.SaveChanges(); this.RunAsync(() => GetCheckCode(checkCode)); } }
private void OldSystemLoginForm_Load(object sender, EventArgs e) { FiddlerApplication.BeforeRequest += oSessions => { //if (oSessions.url.Contains("zhaopin")) oSessions["X-OverrideGateway"] = $"{webProxyIp}:{webProxyPort}"; oSessions.bBufferResponse = true; if (oSessions.url.Contains("captcha.js")) { if (oSessions.RequestHeaders.Exists("If-Modified-Since")) { oSessions.RequestHeaders.Remove("If-Modified-Since"); } if (oSessions.RequestHeaders.Exists("If-None-Match")) { oSessions.RequestHeaders.Remove("If-None-Match"); } if (oSessions.RequestHeaders.Exists("Accept-Encoding")) { oSessions.RequestHeaders.Remove("Accept-Encoding"); } } if (oSessions.RequestHeaders.Exists("User-Agent")) { oSessions.RequestHeaders["User-Agent"] = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.84 Safari/537.36"; } }; FiddlerApplication.BeforeResponse += oSessions => { if (oSessions.url.Contains("captcha.js")) { oSessions.ResponseHeaders["Cache-Control"] = "no-store"; var jsContent = oSessions.GetResponseBodyEncoding().GetString(oSessions.ResponseBody); jsContent = "var hackStr = '';" + jsContent; jsContent = jsContent.Replace("pData.join(\";\")", "hackStr"); jsContent = jsContent.Remove(jsContent.Length - 4); jsContent += "this.hack = function (coordinate){ $(\"#captcha-submitCode\").removeClass(\"btn-disabled\");validate = true;hackStr = coordinate; $(\"#captcha-submitCode\").trigger(\"click\"); return true;}"; jsContent += "}\r\n"; jsContent += "function execHack(coordinate){this.captcha.hack(coordinate); return 1;}"; oSessions.ResponseBody = oSessions.GetResponseBodyEncoding().GetBytes(jsContent); } }; if (!FiddlerApplication.IsStarted()) { FiddlerApplication.Startup(8887, FiddlerCoreStartupFlags.Default); } List <ZhaopinCleaningProcedure> accounts; using (var db = new MangningXssDBEntities()) { accounts = db.ZhaopinCleaningProcedure.Where(w => w.IsEnable && (string.IsNullOrEmpty(w.Cookie) || w.StartTime < DateTime.Today)).ToList(); } this.AsyncSetLog(this.tbx_Log, $"共 {accounts.Count} 个号准备登录!"); this.webBrowser.DocumentCompleted += webBrowser_DocumentCompleted; this.webBrowser.ScriptErrorsSuppressed = true; this.RunAsync(() => { foreach (var item in accounts) { account = item.Account; password = item.Password; webBrowser.Navigate("https://passport.zhaopin.com/org/login"); while (true) { if (!isWaitLogin) { Thread.Sleep(1000); continue; } var index = 0; object obj; var isChecked = false; while (!isChecked) { this.Invoke((MethodInvoker) delegate { index++; if (this.webBrowser.Document?.GetElementById("CheckCodeCapt")?.GetAttribute("value") == "验证通过") { Thread.Sleep(3000); this.webBrowser.Document?.GetElementById("loginbutton")?.InvokeMember("click"); isChecked = true; return; } if (index % 2 != 0) { obj = this.webBrowser.Document?.InvokeScript("execHack", new object[] { "66,76;173,44;239,80" }); } else { obj = this.webBrowser.Document?.InvokeScript("execHack", new object[] { "44,44;190,37;122,48" }); } if (obj == null) { this.AsyncSetLog(this.tbx_Log, "调用验证 JS 异常"); } }); Thread.Sleep(500); } while (true) { if (!isLogined) { Thread.Sleep(1000); continue; } break; } this.Invoke((MethodInvoker) delegate { item.Cookie = webBrowser.Document?.Cookie; var document = webBrowser.Document; document?.ExecCommand("ClearAuthenticationCache", false, null); }); this.AsyncSetLog(this.tbx_Log, $"{account} 获取Cookie成功!"); using (var db = new MangningXssDBEntities()) { var pro = db.ZhaopinCleaningProcedure.FirstOrDefault(f => f.Id == item.Id); if (pro != null) { pro.Cookie = item.Cookie; } db.SaveChanges(); } isWaitLogin = false; isLogined = false; break; } } }); }
/// <summary> /// 标记简历 /// </summary> /// <param name="jsonContent"></param> /// <param name="resumeId"></param> private static void FlagResume(string jsonContent, int resumeId) { using (var db = new MangningXssDBEntities()) { var resume = db.ZhaopinResume.FirstOrDefault(f => f.Id == resumeId); if (resume?.Flag == 0xF) { badoucaiOssClient.DeleteObject(badoucaiBucketName, $"Zhaopin/{resumeId}"); resume.IncludeTime = DateTime.UtcNow; db.SaveChanges(); return; } var resumeIdStr = resumeId.ToString(); if (jsonContent.Contains("detialJSonStr")) { var jsonObj = JsonConvert.DeserializeObject <dynamic>(jsonContent); //var userId = (int)jsonObj.userDetials.userMasterId; if (string.IsNullOrWhiteSpace((string)jsonObj.userDetials.mobilePhone)) { var user = resume != null?db.ZhaopinUser.FirstOrDefault(f => f.Id == resume.UserId && !string.IsNullOrEmpty(f.Cellphone)) : null; if (user == null) { using (var adb = new BadoucaiAliyunDBEntities()) { var reference = adb.CoreResumeReference.AsNoTracking().FirstOrDefault(f => f.Id == resumeIdStr); CoreResumeSummary summary = null; if (reference != null) { summary = adb.CoreResumeSummary.AsNoTracking().FirstOrDefault(f => f.Id == reference.ResumeId); } if (summary != null) { jsonObj.userDetials.mobilePhone = summary.Cellphone; jsonObj.userDetials.email = summary.Email; } } } else { jsonObj.userDetials.mobilePhone = user.Cellphone; jsonObj.userDetials.email = user.Email; } } File.WriteAllText($"{jsonFilePath}{resumeId}", jsonContent); #region 被注释的代码 //var flag = string.IsNullOrEmpty((string)jsonObj.userDetials.mobilePhone) ? (short)0x0 : (short)0xF; //dynamic detialJSonStr; //try //{ // detialJSonStr = jsonObj.detialJSonStr; // if (!string.IsNullOrEmpty((string)jsonObj.detialJSonStr.DateModified)) // { // jsonObj.detialJSonStr = JsonConvert.SerializeObject(jsonObj.detialJSonStr); // } //} //catch (Exception) //{ // detialJSonStr = JsonConvert.DeserializeObject<dynamic>((string)jsonObj.detialJSonStr); //} //if (resume == null) //{ // db.ZhaopinResume.Add(new ZhaopinResume // { // Id = resumeId, // RandomNumber = ((string)jsonObj.resumeNo).Substring(0, 10), // UserId = userId, // RefreshTime = BaseFanctory.GetTime((string)detialJSonStr.DateModified).ToUniversalTime(), // UpdateTime = DateTime.UtcNow, // UserExtId = (string)detialJSonStr.UserMasterExtId, // Source = "XSS", // Flag = flag // }); // handlerFlag = "Insert"; //} //else //{ // resume.RandomNumber = ((string)jsonObj.resumeNo).Substring(0, 10); // resume.UserId = userId; // resume.RefreshTime = BaseFanctory.GetTime((string)detialJSonStr.DateModified).ToUniversalTime(); // resume.UpdateTime = DateTime.UtcNow; // resume.Flag = flag; // handlerFlag = "Update"; //} //db.ZhaopinUser.AddOrUpdate(new ZhaopinUser //{ // Id = userId, // Cellphone = (string)jsonObj.userDetials.mobilePhone, // CreateTime = BaseFanctory.GetTime((string)detialJSonStr.DateCreated).ToUniversalTime(), // Email = (string)jsonObj.userDetials.email, // ModifyTime = BaseFanctory.GetTime((string)detialJSonStr.DateModified).ToUniversalTime(), // Name = (string)jsonObj.userDetials.userName, // Source = "XSS", // UpdateTime = DateTime.UtcNow //}); //var jsonResume = JsonConvert.SerializeObject(jsonObj); //if (flag == 0x0) //{ // var path = $@"F:\ZhaopinOss\Resume\NoInformation\{resumeIdStr.Substring(0, 2)}\{resumeIdStr.Substring(2, 2)}"; // if (!Directory.Exists(path)) Directory.CreateDirectory(path); // File.WriteAllText($@"{path}\{resumeIdStr}",jsonResume); //} //else //{ // var path = $@"F:\ZhaopinOss\Resume\HaveInformation\{resumeIdStr.Substring(0, 2)}\{resumeIdStr.Substring(2, 2)}"; // if (!Directory.Exists(path)) Directory.CreateDirectory(path); // File.WriteAllText($@"{path}\{resumeIdStr}", jsonResume); // using (var stream = new MemoryStream(GZip.Compress(Encoding.UTF8.GetBytes(jsonResume)))) // { // mangningClient.PutObject(mangningBucketName, $"Zhaopin/Resume/{resumeIdStr}", stream); // } //} #endregion badoucaiOssClient.DeleteObject(badoucaiBucketName, $"Zhaopin/{resumeId}"); } else { if (jsonContent.StartsWith("\"<!DOCTYPE HTML>", StringComparison.OrdinalIgnoreCase) || jsonContent.StartsWith("<html", StringComparison.OrdinalIgnoreCase)) { if (jsonContent.StartsWith("\"<!DOCTYPE HTML>", StringComparison.OrdinalIgnoreCase)) { jsonContent = jsonContent.Substring(1); jsonContent = jsonContent.Substring(0, jsonContent.Length - 1); jsonContent = Regex.Unescape(jsonContent); } var updateTime = Regex.Match(jsonContent, "(?s)resumeUpdateTime\">(.+?)<.+?userName.+?alt=\"(.+?)\"").ResultOrDefault("$1", ""); var name = Regex.Match(jsonContent, "(?s)resumeUpdateTime\">(.+?)<.+?userName.+?alt=\"(.+?)\"").ResultOrDefault("$2", "").Replace("\\", "").Replace("/", ""); string fileName; if (string.IsNullOrEmpty(updateTime) || string.IsNullOrEmpty(name)) { fileName = $"{resumeId}"; } else { fileName = $"{name}_{updateTime.Replace("年", "-").Replace("月", "-").Replace("日", "")}.txt"; } File.WriteAllText($"{domFilePath}{fileName}", jsonContent); } else { Trace.WriteLine($"{DateTime.Now} > 简历格式异常!异常简历ID = {resumeId}, Content = {jsonContent}"); } badoucaiOssClient.DeleteObject(badoucaiBucketName, $"Zhaopin/{resumeId}"); } } }
/// <summary> /// 按条件搜索简历 /// </summary> /// <param name="cookieContainer"></param> private static void SearchResumes(CookieContainer cookieContainer) { while (true) { ZhaopinSearchCondition condition; if (count >= refreshCount) { break; } if (!conditionQueue.TryDequeue(out condition)) { Thread.Sleep(100); break; } var pageIndex = 1; var pageTotal = 1; var total = 0; while (pageIndex < pageTotal + 1) { if (count >= refreshCount) { break; } // 2041:宝安 635:南京 653:杭州 639:苏州 765:深圳 var param = string.IsNullOrEmpty(keyword) ? string.Empty : $"SF_1_1_1={keyword}&"; param += string.IsNullOrEmpty(position) ? string.Empty : $"SF_1_1_2={position}&"; param += string.IsNullOrEmpty(industry) ? string.Empty : $"SF_1_1_3={industry}&"; var url = $"https://ihrsearch.zhaopin.com/Home/ResultForCustom?{param}SF_1_1_6={city}&SF_1_1_8={condition.Age}%2C{condition.Age}&SF_1_1_9={condition.Gender}&SF_1_1_5={condition.Degrees}%2C{condition.Degrees}&orderBy=BIRTH_YEAR%2C0&SF_1_1_27=0&SF_1_1_7={date}%2C9&exclude=1&pageIndex={pageIndex}&pageSize=60"; var requestResult = RequestFactory.QueryRequest(url, cookieContainer: cookieContainer, referer: url); if (!requestResult.IsSuccess) { Trace.TraceError($"{DateTime.Now} > Search Error ! Message = {requestResult.ErrorMsg} Condition = {JsonConvert.SerializeObject(condition)}."); continue; } if (pageTotal == 1) { var match = Regex.Match(requestResult.Data, "(?s)<span>(\\d+)</span>份简历.+?rd-resumelist-pageNum\">1/(\\d+)</span>"); if (!match.Success) { if (requestResult.Data.Contains("text/javascript\" r='m'")) { Trace.TraceWarning($"{DateTime.Now} > Cookie Expired !"); } else { Trace.TraceWarning($"{DateTime.Now} > Condition search error ! Page content = {requestResult.Data}"); } return; } total = Convert.ToInt32(match.Result("$1")); pageTotal = Convert.ToInt32(match.Result("$2")); totalCount += total; if (total == 0) { Console.WriteLine($"{DateTime.Now} > Search results are empty !"); break; } } Console.WriteLine($"{DateTime.Now} > 第 {pageIndex} 页 共 {pageTotal} 页 {total} 个结果"); pageIndex++; var matchs = Regex.Matches(requestResult.Data, "(?s)RedirectToRd/([^\r\n]+?)','([^\r\n]+?)','([^\r\n]+?)',this\\);this.+?(\\d{2}-\\d{2}-\\d{2})"); foreach (Match item in matchs) { try { if (count >= refreshCount) { break; } var number = item.Result("$1").Substring(0, 10); DateTime updateDateTime; if (!DateTime.TryParse(item.Result("$4"), out updateDateTime)) { continue; } using (var db = new MangningXssDBEntities()) { var resume = db.ZhaopinResume.FirstOrDefault(f => f.RandomNumber == number); if (resume != null) { if (resume.Flag == 15) { #region 刷新简历更新时间 //if (resume.RefreshTime != null && updateDateTime.Date == resume.RefreshTime.Value.Date) continue; resume.RefreshTime = updateDateTime; resume.Flag = 14; var filePath = $@"D:\Badoucai\Resume\LocationJson\{resume.Id}.json"; using (var stream = new MemoryStream()) { if (!mangningOssClient.DoesObjectExist(mangningBucketName, $"Zhaopin/Resume/{resume.Id}")) { continue; } var bytes = new byte[1024]; int len; var streamContent = mangningOssClient.GetObject(mangningBucketName, $"Zhaopin/Resume/{resume.Id}").Content; while ((len = streamContent.Read(bytes, 0, bytes.Length)) > 0) { stream.Write(bytes, 0, len); } var resumeContent = Encoding.UTF8.GetString(GZip.Decompress(stream.ToArray())); var jsonObj = JsonConvert.DeserializeObject <dynamic>(resumeContent); dynamic detialJSonStr; try { detialJSonStr = jsonObj.detialJSonStr; if (!string.IsNullOrEmpty((string)jsonObj.detialJSonStr.DateModified)) { jsonObj.detialJSonStr = JsonConvert.SerializeObject(jsonObj.detialJSonStr); } } catch (Exception) { detialJSonStr = JsonConvert.DeserializeObject <dynamic>((string)jsonObj.detialJSonStr); } detialJSonStr.DateLastReleased = updateDateTime; detialJSonStr.DateModified = updateDateTime; jsonObj.detialJSonStr = detialJSonStr; var newResumeContent = JsonConvert.SerializeObject(jsonObj); using (var jsonStream = new MemoryStream(GZip.Compress(Encoding.UTF8.GetBytes(newResumeContent)))) { mangningOssClient.PutObject(mangningBucketName, $"Zhaopin/Resume/{resume.Id}", jsonStream); } File.WriteAllText(filePath, newResumeContent); } #endregion } if (resume.Flag == 2) { var user = db.ZhaopinUser.FirstOrDefault(f => f.Id == resume.UserId); if (user == null) { continue; } WatchResumeDetail(item, cookieContainer, user.Cellphone, user.Email); } } else { var incompleteResume = db.ZhaopinIncompleteResume.FirstOrDefault(f => f.ResumeNumber == number); if (incompleteResume == null) { continue; } if (WatchResumeDetail(item, cookieContainer, incompleteResume.Cellphone, incompleteResume.Email)) { incompleteResume.CompletionTime = DateTime.Now; } } db.SaveChanges(); Interlocked.Increment(ref count); if (resume != null) { sb.AppendLine(resume.Id.ToString()); } } } catch (Exception ex) { Trace.TraceError(ex.ToString()); } } Console.WriteLine($"{DateTime.Now} > Total = {totalCount}, Update Count = {count}, Cookie Count = {cookieQueue.Count}, Condition Count = {conditionQueue.Count}. "); } } }
public void Match() { if (!Directory.Exists(savePath)) { Directory.CreateDirectory(savePath); } var cookieQueue = new ConcurrentQueue <KeyValuePair <KeyValuePair <int, string>, CookieContainer> >(); var companyDic = new ConcurrentDictionary <string, int>(); //using (var db = new MangningXssDBEntities()) //{ // var companyArr = db.ZhaoPinCompany.Where(w => w.Source.Contains("MANUAL")).Select(s => s.Id).ToArray(); // var paramArr = db.ZhaopinStaff.Where(w => companyArr.Any(a => a == w.CompanyId) && !string.IsNullOrEmpty(w.Cookie)).Select(s => new { s.CompanyId, s.Username, s.Cookie }).ToArray(); // foreach (var item in paramArr) // { // companyDic[item.Username] = 0; // cookieQueue.Enqueue(new KeyValuePair<KeyValuePair<int, string>, CookieContainer>(new KeyValuePair<int, string>(item.CompanyId, item.Username), item.Cookie.Serialize(".zhaopin.com"))); // } //} using (var db = new MangningXssDBEntities()) { var list = db.ZhaopinCleaningProcedure.Where(w => w.IsEnable /* && !w.IsOnline*/ && !string.IsNullOrEmpty(w.Cookie)); foreach (var item in list) { if (item.StartTime < DateTime.Today) { item.TodayWatch = 0; } companyDic[item.Account] = item.TodayWatch; cookieQueue.Enqueue(new KeyValuePair <KeyValuePair <int, string>, CookieContainer>(new KeyValuePair <int, string>(0, item.Account), item.Cookie.Serialize(".zhaopin.com"))); item.IsOnline = true; item.StartTime = DateTime.Now; } db.SaveChanges(); } Console.WriteLine($"已获取 Cookie 数 => {cookieQueue.Count}"); Task.Run(() => { while (true) { if (searchQueue.Count > 500) { Thread.Sleep(100); continue; } var list = this.GetOldResumes(); foreach (var item in list) { searchQueue.Enqueue(item); } } }); for (var j = 0; j < 16; j++) { Task.Run(() => { try { while (true) { KeyValuePair <KeyValuePair <int, string>, CookieContainer> temp; if (!cookieQueue.TryDequeue(out temp)) { Thread.Sleep(1000); Console.WriteLine($"Company => {temp.Key.Value} 找不到可用Cookie!"); continue; } while (true) { SearchResumeModel search; if (!searchQueue.TryDequeue(out search)) { Console.WriteLine($"Company => {temp.Key.Value} 找不到可搜索的条件!"); Thread.Sleep(1000); continue; } var pageIndex = 1; var pageTotal = 1; var total = 0; var isbreak = false; var gender = search.Gender == "男" ? 1 : 2; var isMatched = false; foreach (var company in search.Companys) { while (pageIndex < pageTotal + 1) { var url = $"https://ihrsearch.zhaopin.com/Home/ResultForCustom?SF_1_1_9={gender}&SF_1_1_25=COMPANY_NAME_ALL:{HttpUtility.UrlEncode(company)}&orderBy=BIRTH_YEAR%2C0&SF_1_1_27=0&exclude=1&pageIndex={pageIndex}&pageSize=60"; var requestResult = RequestFactory.QueryRequest(url, cookieContainer: temp.Value, referer: url); if (!requestResult.IsSuccess) { LogFactory.Warn($"Company => {temp.Key.Value} 条件搜索异常!异常原因=>{requestResult.ErrorMsg} Condition=>{JsonConvert.SerializeObject(search)}"); continue; } if (pageTotal == 1) { var match = Regex.Match(requestResult.Data, "(?s)<span>(\\d+)</span>份简历.+?rd-resumelist-pageNum\">1/(\\d+)</span>"); if (!match.Success) { if (requestResult.Data.Contains("text/javascript\" r='m'")) { LogFactory.Warn($"Cookie 过期,过期用户 => {temp.Key.Value}"); RemoveCookie(temp.Key.Value, companyDic[temp.Key.Value]); } else { LogFactory.Warn($"Company => {temp.Key.Value} 条件搜索异常!返回页面解析异常!{requestResult.Data}"); } isbreak = true; break; } total = Convert.ToInt32(match.Result("$1")); pageTotal = Convert.ToInt32(match.Result("$2")); if (total == 0) { Console.WriteLine("该条件无搜索结果!"); break; } } Console.WriteLine($"Company => {temp.Key.Value} 第 {pageIndex} 页 共 {pageTotal} 页 {total} 个结果"); if (pageTotal > 1) { break; } pageIndex++; var matchs = Regex.Matches(requestResult.Data, "(?s)RedirectToRd/([^\r\n]+?)','([^\r\n]+?)','([^\r\n]+?)',this\\);this.+?(\\d{2}-\\d{2}-\\d{2})"); var index = 0; foreach (Match item in matchs) { var number = item.Result("$1").Substring(0, 10); var numberParam = item.Result("$1").Substring(0, item.Result("$1").IndexOf("?", StringComparison.Ordinal)); var cache = GetResumeOfCache(number); if (cache == null) { DateTime updateDateTime; if (DateTime.TryParse(item.Result("$4"), out updateDateTime)) { QueryResumeIsExists(number, updateDateTime); } requestResult = RequestFactory.QueryRequest($"http://ihr.zhaopin.com/resumesearch/getresumedetial.do?resumeNo={numberParam}&searchresume=1&resumeSource=1&keyword=%E8%AF%B7%E8%BE%93%E5%85%A5%E7%AE%80%E5%8E%86%E5%85%B3%E9%94%AE%E8%AF%8D%EF%BC%8C%E5%A4%9A%E5%85%B3%E9%94%AE%E8%AF%8D%E5%8F%AF%E7%94%A8%E7%A9%BA%E6%A0%BC%E5%88%86%E9%9A%94&t={item.Result("$2")}&k={item.Result("$3")}&v=undefined&version=3&openFrom=1", cookieContainer: temp.Value); if (!requestResult.IsSuccess) { LogFactory.Warn($"Company => {temp.Key.Value} 简历详情查看异常!异常原因=>{requestResult.ErrorMsg} ResumeNumber=>{number}"); continue; } try { var jsonObj = JsonConvert.DeserializeObject <dynamic>(requestResult.Data); if ((int)jsonObj["code"] != 1) { LogFactory.Warn($"Company => {temp.Key.Value} ResumeNumber => {number} 查看详情异常 信息:{(string)jsonObj["message"]} 查看简历份数:{companyDic[temp.Key.Value]}"); if (((string)jsonObj["message"]).Contains("当日查看简历已达上限")) { RemoveCookie(temp.Key.Value, companyDic[temp.Key.Value]); isbreak = true; break; } continue; } var resumeData = jsonObj.data; var userId = (string)resumeData.userDetials.userMasterId; if (search.SearchResumeId.Substring(2, 8) == (userId.Length == 8 ? userId : userId.Substring(1))) { Console.WriteLine($"匹配简历成功!ResumeNumber=>{number} 今日匹配成功数=> {++totalMatched}"); resumeData.userDetials.mobilePhone = search.Cellphone; resumeData.userDetials.email = search.Email; isMatched = true; } SaveCacheByOss(new KeyValuePair <KeyValuePair <int, string>, dynamic>(temp.Key, resumeData)); Console.WriteLine($"Company => {temp.Key.Value} 查看简历成功!查看简历份数:{++companyDic[temp.Key.Value]} ResumeNumber => {number} {++index}/{matchs.Count}"); if (isMatched) { break; } } catch (Exception ex) { while (true) { if (ex.InnerException == null) { break; } ex = ex.InnerException; } LogFactory.Error($"保存简历信息异常!异常信息:{ex.Message} 响应信息:{requestResult.Data}"); } } else { var userId = cache.UserId.ToString(); if (search.SearchResumeId.Substring(2, 8) == (userId.Length == 8 ? userId : userId.Substring(1))) { Console.WriteLine($"匹配简历成功!ResumeNumber=>{number} 今日匹配成功数=> {++totalMatched}"); using (var xdb = new MangningXssDBEntities()) { var model = xdb.ZhaopinMatchedCache.FirstOrDefault(f => f.ResumeId == cache.ResumeId); if (model != null) { model.Cellphone = search.Cellphone; model.Email = search.Email; xdb.SaveChanges(); } } isMatched = true; break; } } } if (isbreak || isMatched) { break; } } if (isbreak || isMatched) { break; } } ChangeResumeStatus(search.SearchResumeId, isMatched); if (isbreak) { break; } } } } catch (Exception ex) { while (true) { if (ex.InnerException == null) { break; } ex = ex.InnerException; } LogFactory.Error($"Watch异常!异常信息:{ex.Message} {ex.StackTrace}"); } }); } }
public void MatchZhaopin() { var resumeQueue = new ConcurrentQueue <OldResumeSummary>(); var cookieQueue = new ConcurrentQueue <QueueParam>(); var todayString = DateTime.Today.ToString("yyyy-MM-dd"); using (var db = new MangningXssDBEntities()) { var companyArr = db.ZhaoPinCompany.Where(w => w.Source.Contains("MANUAL")).Select(s => s.Id).ToArray(); var paramArr = db.ZhaopinStaff.Where(w => companyArr.Any(a => a == w.CompanyId) && !string.IsNullOrEmpty(w.Cookie)).Select(s => new { s.CompanyId, s.Cookie }).ToArray(); foreach (var param in paramArr) { var task = db.ZhaopinResumeMatchLimit.FirstOrDefault(f => f.CompanyId == param.CompanyId); if (task == null) { continue; } var todayTask = db.ZhaopinResumeMatchStatistic.FirstOrDefault(f => f.CompanyId == param.CompanyId && f.Date == todayString); if (todayTask == null) { todayTask = new ZhaopinResumeMatchStatistic { Date = todayString, CompanyId = param.CompanyId, MatchedCount = 0, SearchCount = 0, WatchCount = 0 }; db.ZhaopinResumeMatchStatistic.Add(todayTask); db.SaveChanges(); } if (todayTask.SearchCount == task.DailySearchCount || todayTask.WatchCount == task.DailyWatchCount) { continue; } cookieQueue.Enqueue(new QueueParam { CompanyId = param.CompanyId, Cookie = param.Cookie, DailySeachCount = task.DailySearchCount, DailyWatchCount = task.DailyWatchCount, MatchCount = todayTask.MatchedCount, SeachCount = todayTask.SearchCount, WatchCount = todayTask.WatchCount + todayTask.MatchedCount }); } } Task.Run(() => { while (true) { if (resumeQueue.Count < 10) { using (var db = new ResumeMatchDBEntities()) { var resumeList = db.OldResumeSummary.Where(w => w.ResumeId.Length == 10 && !w.IsMatched && w.Status == 0).Take(1000).ToList(); resumeList.ForEach(f => { resumeQueue.Enqueue(f); }); } } Thread.Sleep(TimeSpan.FromSeconds(1)); } }); for (var j = 0; j < 16; j++) { Task.Run(() => { using (var db = new ResumeMatchDBEntities()) { while (true) { #region 匹配 OldResumeSummary resume; if (!resumeQueue.TryDequeue(out resume)) { continue; } var startNum = 0; var resumeTemp = resume; resume = db.OldResumeSummary.FirstOrDefault(f => f.Id == resumeTemp.Id); QueueParam cookie; if (!cookieQueue.TryDequeue(out cookie)) { return; } try { using (var xdb = new MangningXssDBEntities()) { var filePath = $@"E:\Data\智联招聘\{resume.Template}\{resume.ResumeId}.{Path.GetFileNameWithoutExtension(resume.Template)}"; if (!File.Exists(filePath)) { LogFactory.Warn($"指定路径不存在!ResumeNumber=>{resume.ResumeId} Path=>{filePath}"); resume.Status = 1; resume.MatchTime = DateTime.Now; db.SaveChanges(); continue; } var sourceCode = File.ReadAllText(filePath); //var sourceCode = string.Empty; var genderMatch = Regex.Match(sourceCode, "(男|女)"); var gender = string.Empty; if (genderMatch.Success) { gender = genderMatch.Value == "男" ? "1" : "2"; } var matchs = Regex.Matches(sourceCode, "[\u4e00-\u9fa5]{4,11}有限公司[\u4e00-\u9fa5]{0,6}"); var companys = new List <string>(); if (matchs.Count == 0) { if (cookie.SeachCount - cookie.WatchCount < 50) { Console.WriteLine($"该简历未匹配到公司!ResumeNumber=> {resume.ResumeId}"); resume.Status = 1; resume.MatchTime = DateTime.Now; db.SaveChanges(); cookieQueue.Enqueue(cookie); continue; } companys.Add(string.Empty); } for (var i = 0; i < matchs.Count; i++) { if (i == 2) { break; } companys.Add(matchs[i].Value); } var isMatched = true; var age = string.Empty; for (var i = 0; i < companys.Count; i++) { var companyName = companys[i]; var statistic = xdb.ZhaopinResumeMatchStatistic.FirstOrDefault(f => f.Date == todayString && f.CompanyId == cookie.CompanyId); if (statistic == null) { continue; } var cookieContainer = cookie.Cookie.Serialize(".zhaopin.com"); var start = string.IsNullOrEmpty(companyName) ? startNum * 100 : 0; var paramDictionary = new Dictionary <string, string> { { "keywords", "的" }, { "startNum", $"{start}" }, { "rowsCount", "100" }, { "sortColumnName", "sortUpDate" }, { "sortColumn", "sortUpDate desc" }, { "onlyHasImg", "false" }, { "anyKeyWord", "false" }, { "sex", gender }, { "companyName", companyName }, { "onlyLastWork", "false" }, { "ageStart", age }, { "ageEnd", age } }; var requestResult = HttpClientFactory.RequestForString("https://ihr.zhaopin.com/resumesearch/search.do", HttpMethod.Post, paramDictionary, cookieContainer); if (!requestResult.IsSuccess) { isMatched = false; break; } var jObject = JsonConvert.DeserializeObject(requestResult.Data) as JObject; if (jObject["code"] != null) { LogFactory.Warn($"CompanyId => {cookie.CompanyId} 搜索异常 异常信息:{(string)jObject["message"]}"); isMatched = false; return; } statistic.SearchCount += 1; ++cookie.SeachCount; Console.WriteLine($"CompanyId => {cookie.CompanyId} 搜索公司=> {companyName} 性别=> {genderMatch.Value} 今日请求次数:{cookie.SeachCount} "); xdb.SaveChanges(); var matchResumes = (JArray)jObject?["results"]; if (matchResumes == null) { continue; } if (matchResumes.Count == 0) { continue; } var matchResume = matchResumes.FirstOrDefault(a => ((string)a["number"]).Substring(0, 10) == resume.ResumeId); var resumes = matchResumes.Where(w => DateTime.Parse((string)w["modifyDate"]) > DateTime.Today.AddDays(-2)); foreach (var item in resumes) { var wcount = matchResume == null ? cookie.WatchCount : cookie.WatchCount + 1; if (cookie.SeachCount <= wcount) { break; } var number = ((string)item["number"]).Substring(0, 10); if (xdb.ZhaopinResume.Any(a => a.RandomNumber == number)) { continue; } lock (lockObj) { if (xdb.ZhaopinWatchedResume.Any(a => a.ResumeNumber == number)) { continue; } requestResult = HttpClientFactory.RequestForString($"http://ihr.zhaopin.com/resumesearch/getresumedetial.do?resumeNo={(string)item["id"]}_1&resumeSource=1&key=&{(string)item["valResumeTimeStr"]}", HttpMethod.Get, null, cookieContainer); if (requestResult.IsSuccess) { var resumeData = JsonConvert.DeserializeObject <dynamic>(requestResult.Data).data; var resumeDetail = JsonConvert.DeserializeObject(resumeData.detialJSonStr.ToString()); var resumeid = resumeData.resumeId != null ? (int)resumeData.resumeId : resumeDetail.ResumeId != null ? (int)resumeDetail.ResumeId : 0; statistic.WatchCount += 1; Console.WriteLine($"CompanyId => {cookie.CompanyId} 查看简历成功!查看简历份数:{++cookie.WatchCount} ResumeNumber => {number}"); xdb.ZhaopinWatchedResume.Add(new ZhaopinWatchedResume { Id = resumeid, ResumeNumber = number, CompanyId = cookie.CompanyId, WatchTime = DateTime.UtcNow }); xdb.SaveChanges(); } } } if (cookie.SeachCount - cookie.WatchCount > 50 && string.IsNullOrEmpty(companyName) && matchResume == null) { i--; if (startNum == 40) { if (string.IsNullOrEmpty(age)) { age = "18"; } else { age = (Convert.ToInt32(age) + 1).ToString(); } startNum = 0; } else { startNum++; } continue; } if (matchResume == null) { continue; } var resumeNo = (string)matchResume["id"] + "_1"; var resumeNumber = ((string)matchResume["number"]).Substring(0, 10); if (xdb.ZhaopinResume.Any(a => a.RandomNumber == resumeNumber)) { continue; } var valResumeTimeStr = (string)matchResume["valResumeTimeStr"]; requestResult = HttpClientFactory.RequestForString($"http://ihr.zhaopin.com/resumesearch/getresumedetial.do?resumeNo={resumeNo}&resumeSource=1&key=&{valResumeTimeStr}", HttpMethod.Get, null, cookieContainer); if (!requestResult.IsSuccess) { continue; } ++cookie.WatchCount; var resumeJson = JsonConvert.DeserializeObject <dynamic>(requestResult.Data).data; var detail = JsonConvert.DeserializeObject(resumeJson.detialJSonStr.ToString()); var resumeId = resumeJson.resumeId != null ? (int)resumeJson.resumeId : detail.ResumeId != null ? (int)detail.ResumeId : 0; if (xdb.ZhaopinMatchedResume.Any(a => a.Id == resumeId)) { statistic.MatchedCount += 1; LogFactory.Warn($"匹配到重复简历!ResumeId=>{resumeId} ResumeNumber=>{resumeNumber}"); continue; } if (resumeId == 0) { LogFactory.Warn($"CompanyId => {cookie.CompanyId} 解析异常!ResumeId 为空, ResumeNumber:{resumeNumber}"); continue; } var userId = (int)resumeJson.userDetials.userMasterId; var user = xdb.ZhaopinUser.FirstOrDefault(f => f.Id == userId); if (user != null) { if (!user.Source.Contains("MANUAL")) { user.Id = userId; user.Source = "XSS"; user.ModifyTime = BaseFanctory.GetTime((string)detail.DateModified).ToUniversalTime(); user.CreateTime = BaseFanctory.GetTime((string)detail.DateCreated).ToUniversalTime(); user.Cellphone = resume.Cellphone; user.Email = resume.Email; user.Name = resumeJson.userDetials.userName.ToString(); user.UpdateTime = DateTime.UtcNow; user.Username = resumeJson.userDetials.email.ToString(); } } else { xdb.ZhaopinUser.Add(new ZhaopinUser { Id = userId, Source = "XSS", ModifyTime = BaseFanctory.GetTime((string)detail.DateModified).ToUniversalTime(), CreateTime = BaseFanctory.GetTime((string)detail.DateCreated).ToUniversalTime(), Cellphone = resume.Cellphone, Email = resume.Email, Name = resumeJson.userDetials.userName.ToString(), UpdateTime = DateTime.UtcNow }); } var resumeEntity = xdb.ZhaopinResume.FirstOrDefault(f => f.Id == resumeId); if (resumeEntity == null) { xdb.ZhaopinResume.Add(new ZhaopinResume { Id = resumeId, RandomNumber = resumeNumber, UserId = userId, RefreshTime = BaseFanctory.GetTime((string)detail.DateModified).ToUniversalTime(), UpdateTime = DateTime.UtcNow, UserExtId = detail.UserMasterExtId.ToString(), DeliveryNumber = resumeEntity?.DeliveryNumber, Source = resumeEntity == null ? "XSS" : resumeEntity.Source }); } else { resumeEntity.RandomNumber = resumeNumber; resumeEntity.UserId = userId; resumeEntity.RefreshTime = BaseFanctory.GetTime((string)detail.DateModified).ToUniversalTime(); resumeEntity.UpdateTime = DateTime.UtcNow; resumeEntity.UserExtId = detail.UserMasterExtId.ToString(); resumeEntity.DeliveryNumber = resumeEntity?.DeliveryNumber; resumeEntity.Source = resumeEntity == null ? "XSS" : resumeEntity.Source; } var path = $"{ConfigurationManager.AppSettings["Resume.SavePath"]}{DateTime.Now:yyyyMMdd}"; if (!Directory.Exists(path)) { Directory.CreateDirectory(path); } var resumePath = $@"{path}\{resumeId}.json"; resumeJson.userDetials.email = resume.Email; resumeJson.userDetials.mobilePhone = resume.Cellphone; File.WriteAllText(resumePath, JsonConvert.SerializeObject(resumeJson)); uploadOssActionBlock.Post(resumePath); statistic.MatchedCount += 1; Console.WriteLine($"CompanyId => {cookie.CompanyId} 搜索简历成功!匹配成功 {++cookie.MatchCount} 份, ResumeNumner=>{resumeNumber},ResumeId=>{resumeId}"); xdb.ZhaopinMatchedResume.Add(new ZhaopinMatchedResume { Id = resumeId, CompanyId = cookie.CompanyId, MatchTime = DateTime.UtcNow }); xdb.SaveChanges(); resume.IsMatched = true; break; } if (cookie.SeachCount < cookie.DailySeachCount) { cookieQueue.Enqueue(cookie); } if (isMatched) { resume.Status = 99; resume.MatchTime = DateTime.Now; } db.SaveChanges(); } } catch (Exception ex) { //while (true) //{ // if(ex.InnerException == null) break; // ex = ex.InnerException; //} if (cookie.SeachCount < cookie.DailySeachCount) { cookieQueue.Enqueue(cookie); } LogFactory.Warn($"程序异常, 异常信息:{ex.Message} 堆栈:{ex.StackTrace}"); } #endregion } } }); } }
/// <summary> /// 处理简历 /// </summary> /// <param name="item"></param> /// <param name="cellphone"></param> /// <param name="email"></param> /// <param name="cleaningProcedure"></param> /// <param name="businessId"></param> private static void HandleResume(Match item, string cellphone, string email, ZhaopinCleaningProcedure cleaningProcedure, int businessId) { var number = item.Result("$1").Substring(0, 10); DateTime updateDateTime; if (!DateTime.TryParse(item.Result("$4"), out updateDateTime)) { return; } using (var db = new MangningXssDBEntities()) { var resume = db.ZhaopinResume.FirstOrDefault(f => f.RandomNumber == number); if (resume != null) { if (resume.Flag == 15) { #region 刷新简历更新时间 //if (resume.RefreshTime != null && updateDateTime.Date == resume.RefreshTime.Value.Date) continue; resume.RefreshTime = updateDateTime; resume.Flag = 14; var filePath = $@"D:\Badoucai\Resume\LocationJson\{resume.Id}.json"; using (var stream = new MemoryStream()) { if (!mangningOssClient.DoesObjectExist(mangningBucketName, $"Zhaopin/Resume/{resume.Id}")) { return; } var bytes = new byte[1024]; int len; var streamContent = mangningOssClient.GetObject(mangningBucketName, $"Zhaopin/Resume/{resume.Id}").Content; while ((len = streamContent.Read(bytes, 0, bytes.Length)) > 0) { stream.Write(bytes, 0, len); } var resumeContent = Encoding.UTF8.GetString(GZip.Decompress(stream.ToArray())); var jsonObj = JsonConvert.DeserializeObject <dynamic>(resumeContent); dynamic detialJSonStr; try { detialJSonStr = jsonObj.detialJSonStr; if (!string.IsNullOrEmpty((string)jsonObj.detialJSonStr.DateModified)) { jsonObj.detialJSonStr = JsonConvert.SerializeObject(jsonObj.detialJSonStr); } } catch (Exception) { detialJSonStr = JsonConvert.DeserializeObject <dynamic>((string)jsonObj.detialJSonStr); } detialJSonStr.DateLastReleased = updateDateTime; detialJSonStr.DateModified = updateDateTime; jsonObj.detialJSonStr = detialJSonStr; var newResumeContent = JsonConvert.SerializeObject(jsonObj); using (var jsonStream = new MemoryStream(GZip.Compress(Encoding.UTF8.GetBytes(newResumeContent)))) { mangningOssClient.PutObject(mangningBucketName, $"Zhaopin/Resume/{resume.Id}", jsonStream); } File.WriteAllText(filePath, newResumeContent); } #endregion } if (resume.Flag == 2) { var user = db.ZhaopinUser.FirstOrDefault(f => f.Id == resume.UserId); if (user == null) { return; } if (!WatchResumeDetail(item, cookieContainer, user.Cellphone, user.Email, businessId)) { var procedure = db.ZhaopinCleaningProcedure.FirstOrDefault(f => f.Id == cleaningProcedure.Id); if (procedure != null) { procedure.Cookie = null; db.SaveChanges(); } businessIdQueue.Enqueue(businessId); } } } else { var incompleteResume = db.ZhaopinIncompleteResume.FirstOrDefault(f => f.ResumeNumber == number); if (incompleteResume != null) { incompleteResume.CompletionTime = DateTime.Now; } WatchResumeDetail(item, cookieContainer, cellphone, email, businessId); } db.SaveChanges(); } }
/// <summary> /// 处理多迪简历 /// </summary> private static void HandleDodiResume() { while (true) { ZhaopinCleaningProcedure cleaningProcedure; if (!cookieQueue.TryDequeue(out cleaningProcedure)) { Thread.Sleep(100); continue; } var localCookieContainer = cleaningProcedure.Cookie.Serialize(".zhaopin.com"); while (true) { int businessId; if (!businessIdQueue.TryDequeue(out businessId)) { Thread.Sleep(100); continue; } using (var db = new MangningXssDBEntities()) { var dodiBusiness = db.DodiBusiness.FirstOrDefault(f => f.Id == businessId); if (dodiBusiness != null) { dodiBusiness.Status = 2; db.SaveChanges(); } } try { var response = RequestFactory.QueryRequest($"http://crm.dodi.cn/index.php/Main/khxxy/business_id/{businessId}/source/false_note", cookieContainer: cookieContainer); if (!response.IsSuccess) { continue; } if (!response.Data.Contains("商 机 ID:")) { Trace.WriteLine($"{DateTime.Now} > Business ID is Empty ! BusinessId = {businessId}"); continue; } var match = Regex.Match(response.Data, @"resume_email\('(.*?)','(\d+)','(\d+)','(\d+)',(\d+)\)"); if (!match.Success) { Trace.WriteLine($"{DateTime.Now} > Details match failed ! BusinessId = {businessId}"); continue; } var email = HttpUtility.UrlEncode(match.Result("$1")); var phone = HttpUtility.UrlEncode(match.Result("$2")); var email_id = HttpUtility.UrlEncode(match.Result("$3")); var now_month = HttpUtility.UrlEncode(match.Result("$4")); var school_id = HttpUtility.UrlEncode(match.Result("$5")); response = HttpClientFactory.RequestForString($"http://crm.dodi.cn/index.php/Main/email_body?email={email}&phone={phone}&email_id={email_id}&now_month={now_month}&school_id={school_id}", HttpMethod.Get, null, cookieContainer); if (!response.IsSuccess) { Trace.WriteLine($"{DateTime.Now} > Details request failed ! BusinessId = {businessId}"); continue; } var resumeDetail = Regex.Unescape(response.Data); var company = Regex.Match(resumeDetail, "20[01]\\d[.][\\d\\s-.]+(.*?)\\s*(").ResultOrDefault("$1", ""); if (string.IsNullOrEmpty(company)) { Console.WriteLine($"{DateTime.Now} > Company is Empty ! BusinessId = {businessId}"); continue; } var matchs = Regex.Matches(resumeDetail, "20[01]\\d\\.\\d{2}\\s-\\s20[01]\\d\\.\\d{2}\\s+([^&]+?)&"); var project = string.Empty; foreach (Match item in matchs) { var matchString = item.Result("$1"); if (matchString.Contains("(")) { continue; } if (matchString.Contains("</br>")) { continue; } if (matchString.Contains(" ")) { continue; } project = matchString; break; } if (string.IsNullOrEmpty(project)) { Console.WriteLine($"{DateTime.Now} > Project is Empty ! BusinessId = {businessId}"); continue; } var url = $"https://ihrsearch.zhaopin.com/Home/ResultForCustom?SF_1_1_1={HttpUtility.UrlEncode(project)}&SF_1_1_25=COMPANY_NAME_ALL:{HttpUtility.UrlEncode(company)}&SF_1_1_27=0&orderBy=DATE_MODIFIED,1&pageSize=60&exclude=1"; var requestResult = RequestFactory.QueryRequest(url, cookieContainer: localCookieContainer, referer: url); if (!requestResult.IsSuccess) { Trace.TraceError($"{DateTime.Now} > Search request failed ! BusinessId = {businessId}."); continue; } match = Regex.Match(requestResult.Data, "(?s)<span>(\\d+)</span>份简历.+?rd-resumelist-pageNum\">1/(\\d+)</span>"); if (!match.Success) { if (requestResult.Data.Contains("text/javascript\" r='m'")) { Console.WriteLine($"{DateTime.Now} > Cookie Expired ! Account = {cleaningProcedure.Account}."); } else { Trace.TraceWarning($"{DateTime.Now} > Condition search error ! Page content = {requestResult.Data}, Account = {cleaningProcedure.Account}."); } using (var db = new MangningXssDBEntities()) { var procedure = db.ZhaopinCleaningProcedure.FirstOrDefault(f => f.Id == cleaningProcedure.Id); if (procedure != null) { procedure.Cookie = null; db.SaveChanges(); } } businessIdQueue.Enqueue(businessId); break; } var total = Convert.ToInt32(match.Result("$1")); if (total != 1) { Console.WriteLine($"{DateTime.Now} > Search match wrong ! ResultCount = {total}, BusinessId = {businessId}, Company = {company}, Project = {project}."); continue; } match = Regex.Match(requestResult.Data, "(?s)RedirectToRd/([^\r\n]+?)','([^\r\n]+?)','([^\r\n]+?)',this\\);this.+?(\\d{2}-\\d{2}-\\d{2})"); HandleResume(match, HttpUtility.HtmlDecode(phone), HttpUtility.HtmlDecode(email), cleaningProcedure, businessId); } catch (Exception ex) { Trace.TraceError(ex.ToString()); } } } }
/// <summary> /// 下载投递简历列表 /// </summary> /// <param name="path"></param> /// <param name="cookieStr"></param> public void DownOldSystemResumeDetail(string path, string cookieStr) { var count = 0; while (true) { var requestResult = RequestFactory.QueryRequest("https://rd2.zhaopin.com/rdapply/resumes/apply/search?SF_1_1_38=7,9&orderBy=CreateTime", "PageList2=&DColumn_hidden=&searchKeyword=&curSubmitRecord=1586&curMaxPageNum=80&PageList2=1&buttonAsse=%E5%AF%BC%E5%85%A5%E6%B5%8B%E8%AF%84%E7%B3%BB%E7%BB%9F&buttonInfo=%E5%8F%91%E9%80%9A%E7%9F%A5%E4%BF%A1&SF_1_1_50=1&SF_1_1_51=-1&SF_1_1_45=&SF_1_1_44=&SF_1_1_52=0&SF_1_1_49=0&IsInvited=0&position_city=%5B%25%25POSITION_CITY%25%25%5D&deptName=&select_unique_id=&selectedResumeList=&PageNo=&PosState=&MinRowID=&MaxRowID=2722819791&RowsCount=123&PagesCount=5&PageType=0&CurrentPageNum=1&Position_IDs=%5B%25%25POSITION_IDS%25%25%5D&Position_ID=%5B%25%25POSITION_ID%25%25%5D&SortType=0&isCmpSum=0&SelectIndex_Opt=0&Resume_count=0&CID=56211453&forwardingEmailList=&click_search_op_type=-1&X-Requested-With=XMLHttpRequest", RequestEnum.POST, _cookieContainer); if (!requestResult.IsSuccess) { this.AsyncSetLog(this.tbx_Log, "请求异常!异常原因:" + requestResult.ErrorMsg); return; } var matches = Regex.Matches(requestResult.Data, "(?s)javascript:ViewOneResume.+?'(\\d+)'\\);\" href=\"(.+?)\".+?>(.+?)</a.+?<td title=\"(.+?)\""); if (requestResult.Data.Contains("因请求量过大导致系统无法处理您的请求,您需要通过验证才能继续后续的操作!")) { this.AsyncSetLog(this.tbx_Log, "搜索简历列表失败!出现验证码!"); using (var db = new MangningXssDBEntities()) { if (!db.ZhaopinCheckCode.Any(a => a.Account == companyName && (a.Status == 0 || a.Status == 1))) { db.ZhaopinCheckCode.Add(new ZhaopinCheckCode { Account = companyName, Cookie = cookieStr, CreateTime = DateTime.Now, Status = 0, Type = 1 }); db.SaveChanges(); } } Thread.Sleep(retryInterval); continue; } if (matches.Count == 0) { Thread.Sleep(10 * 1000); continue; } var ids = string.Empty; foreach (Match match in matches) { try { var filePath = $"{path}\\{match.Result("$3").Replace("\\", "").Replace("/", "")}_{DateTime.Parse(match.Result("$4")):yyyy-MM-dd HH:mm:ss}.txt"; while (true) { requestResult = RequestFactory.QueryRequest($"https:{match.Result("$2")}", cookieContainer: _cookieContainer); if (!requestResult.IsSuccess) { this.AsyncSetLog(this.tbx_Log, "请求异常!异常原因:" + requestResult.ErrorMsg); return; } if (requestResult.Data.Contains("因请求量过大导致系统无法处理您的请求,您需要通过验证才能继续后续的操作!")) { this.AsyncSetLog(this.tbx_Log, "查看简历详情失败!出现验证码!"); using (var db = new MangningXssDBEntities()) { if (!db.ZhaopinCheckCode.Any(a => a.Account == companyName && (a.Status == 0 || a.Status == 1))) { db.ZhaopinCheckCode.Add(new ZhaopinCheckCode { Account = companyName, Cookie = cookieStr, CreateTime = DateTime.Now, Status = 0, Type = 1 }); db.SaveChanges(); } } Thread.Sleep(retryInterval); continue; } Thread.Sleep(interval); ids += match.Result("$1") + "%3B"; break; } File.WriteAllText(filePath, requestResult.Data); this.AsyncSetLog(this.tbx_Log, $"下载成功!第 {++count} 份 Path => {filePath}"); var htmlDocument = new HtmlDocument(); htmlDocument.LoadHtml(requestResult.Data); resumeQueue.Enqueue(new KeyValuePair <HtmlDocument, DateTime>(htmlDocument, DateTime.Parse(match.Result("$4")))); } catch (Exception ex) { this.AsyncSetLog(this.tbx_Log, "程序异常!异常消息:" + ex.Message); } } requestResult = RequestFactory.QueryRequest("https://rd2.zhaopin.com/RdApply/Resumes/Apply/SetResumeState", $"ids={ids}&oldResumeState=1&resumeState=4", RequestEnum.POST, _cookieContainer); var jsonObj = JsonConvert.DeserializeObject <dynamic>(requestResult.Data); if ((int)jsonObj.Code != 200) { this.AsyncSetLog(this.tbx_Log, $"标记简历为不合适失败!Message = {(string)jsonObj.Message}"); continue; } this.AsyncSetLog(this.tbx_Log, "标记简历为不合适成功!"); } this.AsyncSetLog(this.tbx_Log, "简历处理完成!"); this.RunInMainthread(() => { this.btn_StartDownload.Enabled = true; }); }
/// <summary> /// 查看简历详情 /// </summary> /// <param name="match"></param> /// <param name="localCookieContainer"></param> /// <param name="cellphone"></param> /// <param name="email"></param> /// <param name="businessId"></param> /// <returns></returns> private static bool WatchResumeDetail(Match match, CookieContainer localCookieContainer, string cellphone, string email, int businessId) { var number = match.Result("$1").Substring(0, 10); var numberParam = match.Result("$1").Substring(0, match.Result("$1").IndexOf("?", StringComparison.Ordinal)); var requestResult = RequestFactory.QueryRequest($"http://ihr.zhaopin.com/resumesearch/getresumedetial.do?resumeNo={numberParam}&searchresume=1&resumeSource=1&keyword=%E8%AF%B7%E8%BE%93%E5%85%A5%E7%AE%80%E5%8E%86%E5%85%B3%E9%94%AE%E8%AF%8D%EF%BC%8C%E5%A4%9A%E5%85%B3%E9%94%AE%E8%AF%8D%E5%8F%AF%E7%94%A8%E7%A9%BA%E6%A0%BC%E5%88%86%E9%9A%94&t={match.Result("$2")}&k={match.Result("$3")}&v=undefined&version=3&openFrom=1", cookieContainer: localCookieContainer); if (!requestResult.IsSuccess) { Trace.WriteLine($"{DateTime.Now} > Watching error ! Message = {requestResult.ErrorMsg}, ResumeNumber = {number}"); return(false); } try { var jsonObj = JsonConvert.DeserializeObject <dynamic>(requestResult.Data); if ((int)jsonObj["code"] != 1) { Trace.WriteLine($"{DateTime.Now} > Watching failure ! Message = {(string)jsonObj["message"]}, ResumeNumber = {number}"); return(false); } var resumeData = jsonObj.data; resumeData.userDetials.mobilePhone = cellphone; resumeData.userDetials.email = email; var resumeDetail = JsonConvert.DeserializeObject(resumeData.detialJSonStr.ToString()); var resumeId = resumeData.resumeId != null ? (int)resumeData.resumeId : resumeDetail.ResumeId != null ? (int)resumeDetail.ResumeId : 0; using (var db = new MangningXssDBEntities()) { var business = db.DodiBusiness.FirstOrDefault(f => f.Id == businessId); if (business != null) { business.Status = 3; db.SaveChanges(); } } File.WriteAllText($"{path}{resumeId}.json", JsonConvert.SerializeObject(resumeData)); Console.WriteLine($"{DateTime.Now} > Completion success ! ResumeNumber = {number}."); return(true); } catch (Exception ex) { while (true) { if (ex.InnerException == null) { break; } ex = ex.InnerException; } Trace.WriteLine($"{DateTime.Now} > Watching exception ! Message = {ex.Message}, ResumeNumber = {number}"); return(false); } }
public void Buquan() { var queue = new ConcurrentQueue <KeyValuePair <int, KeyValuePair <string, DateTime> > >(); var index = 0; Task.Run(() => { while (true) { KeyValuePair <int, KeyValuePair <string, DateTime> > temp; if (!queue.TryDequeue(out temp)) { Thread.Sleep(10); continue; } using (var db = new MangningXssDBEntities()) { var resume = db.ZhaopinResume.FirstOrDefault(f => f.Id == temp.Key); if (resume == null) { LogFactory.Warn($"找不到简历 ResumeId =>{temp.Key}"); continue; } var user = db.ZhaopinUser.FirstOrDefault(f => f.Id == resume.UserId); if (user == null) { LogFactory.Warn($"找不到用户 ResumeId =>{temp.Key} UserId =>{resume.UserId}"); continue; } if (string.IsNullOrEmpty(temp.Value.Key) && string.IsNullOrEmpty(user.Cellphone)) { db.ZhaopinResume.Remove(resume); db.ZhaopinUser.Remove(user); Console.WriteLine($"删除成功!ResumeId=>{temp.Key} {++index}/{buquanTotal}"); } else { resume.UserExtId = temp.Value.Key; user.CreateTime = temp.Value.Value; Console.WriteLine($"补全成功!{++index}/{buquanTotal}"); } db.SaveChanges(); } } }); using (var db = new MangningXssDBEntities()) { var idArray = db.ZhaopinResume.Where(w => w.Source == "Watch").Select(s => s.Id).ToArray(); buquanTotal = idArray.Length; foreach (var businessId in idArray) { try { var streamContent = newOss.GetObject(newBucket, $"WatchResume/{businessId}").Content; using (var stream = new MemoryStream()) { var bytes = new byte[1024]; int len; while ((len = streamContent.Read(bytes, 0, bytes.Length)) > 0) { stream.Write(bytes, 0, len); } var jsonContent = Encoding.UTF8.GetString(GZip.Decompress(stream.ToArray())); var resumeData = JsonConvert.DeserializeObject <dynamic>(jsonContent); var resumeDetail = JsonConvert.DeserializeObject <dynamic>((string)resumeData.detialJSonStr); var createDate = BaseFanctory.GetTime((string)resumeDetail.DateCreated).ToUniversalTime(); var extId = resumeDetail.UserMasterExtId.ToString(); queue.Enqueue(new KeyValuePair <int, KeyValuePair <string, DateTime> >(businessId, new KeyValuePair <string, DateTime>(extId, createDate))); } } catch (Exception ex) { LogFactory.Warn($"Oss 获取简历异常! ResumeId => {businessId} 异常原因 => {ex.Message}"); queue.Enqueue(new KeyValuePair <int, KeyValuePair <string, DateTime> >(businessId, new KeyValuePair <string, DateTime>("", DateTime.Now))); } } } }
/// <summary> /// 上传简历 /// </summary> private static void UploadResume() { var serializer = new Serialization.Template.Zhaopin.Json.v1.Serializer(); var stopwatch = new Stopwatch(); while (true) { string path; if (!uploadQueue.TryDequeue(out path)) { Thread.Sleep(100); continue; } Interlocked.Increment(ref totalUpload); try { stopwatch.Restart(); var resumeId = Convert.ToInt32(Path.GetFileNameWithoutExtension(path)); var content = File.ReadAllText(path); var jsonObj = JsonConvert.DeserializeObject <dynamic>(content); dynamic formatterResume; try { if (jsonObj.Flag != null && jsonObj.Flag < 0) { formatterResume = Format.Convert_V0(Format.ConvertTo_Dtl_V5(content)); } else { var serializationResume = serializer.Deserialize(content); formatterResume = Formatter.Template.Zhaopin.Json.v1.Formatter.Format(serializationResume); } } catch (Exception ex) { var filePath = $"{ConfigurationManager.AppSettings["File.FailPath"]}{Path.GetFileName(path)}"; if (File.Exists(filePath)) { File.Delete(filePath); } File.Move(path, filePath); Trace.TraceError(ex.ToString()); using (var db = new MangningXssDBEntities()) { var resume = db.ZhaopinResume.FirstOrDefault(f => f.Id == resumeId); if (resume != null) { resume.Flag = 0xA; } db.SaveChanges(); } continue; } string tag; while (true) { var data = JsonConvert.SerializeObject(new { formatterResume.Reference.Id, formatterResume.Reference.Source }); var response = PrepareUploadResume(data); if (response.Code.ToString() != "0") { FinishUploadResume(data); Trace.WriteLine($"{DateTime.Now} > PrepareUploadResume failed ! Data = {data}, Response = {JsonConvert.SerializeObject(response)}"); continue; } var badoucaiResumeJson = JsonConvert.SerializeObject(formatterResume); response = UploadResume(badoucaiResumeJson); var returnCode = (string)response.Code; using (var db = new MangningXssDBEntities()) { tag = response.Reference?.Tag.ToString(); db.ZhaopinResumeUploadLog.Add(new ZhaopinResumeUploadLog { ResumeId = resumeId, ReturnCode = returnCode, Tag = tag, UploadTime = DateTime.Now }); FinishUploadResume(data); var resume = db.ZhaopinResume.FirstOrDefault(f => f.Id == resumeId); if (resume != null) { resume.Flag = 0xF; } if (jsonObj.Flag != null && jsonObj.Flag < 0) { resume.Flag = 0xB; } if (returnCode != "0") { Trace.WriteLine($"{DateTime.Now} > UploadResume failed !ResumeId = {resumeId}, Response = {JsonConvert.SerializeObject(response)}."); tag = "NULL"; db.SaveChanges(); var filePath = $"{ConfigurationManager.AppSettings["File.UploadFailPath"]}{Path.GetFileName(path)}"; if (File.Exists(filePath)) { File.Delete(filePath); } File.Move(path, filePath); break; } if (tag == "C" || tag == "U" || tag == "N") { using (var stream = new MemoryStream(GZip.Compress(Encoding.UTF8.GetBytes(badoucaiResumeJson)))) { var id = response.Reference.ResumeId.ToString(); badoucaiOssClient.PutObject(badoucaiBucketName, $"Badoucai/{id}", stream); } ProgramTasksThread.HandleResume(content, resumeId); // 更新 Resume 表的信息 } File.Delete(path); db.SaveChanges(); Interlocked.Increment(ref successUpload); } break; } stopwatch.Stop(); Console.WriteLine($"{DateTime.Now} > ResumeId = {resumeId}, Tag = {tag}, {successUpload}/{totalUpload} = {Math.Round(successUpload / (double)totalUpload, 3) * 100}%, Elapsed = {stopwatch.ElapsedMilliseconds} ms."); } catch (Exception ex) { Trace.TraceError(ex.ToString()); } } }
public void Watch() { //using (var db = new MangningXssDBEntities()) //{ // var maxId = db.ZhaopinCleaningProcedure.Max(m => m.Id); // for (var i = 1; i <= 11; i++) // { // db.ZhaopinCleaningProcedure.Add(new ZhaopinCleaningProcedure // { // Id = (short)(i + maxId), // Account = $"qhzl_{i:000}", // Cookie = null, // IsEnable = true, // IsOnline = false, // Password = "******", // StartTime = DateTime.Now.AddDays(-1) // }); // } // db.SaveChanges(); //} if (!Directory.Exists(savePath)) { Directory.CreateDirectory(savePath); } var cookieQueue = new ConcurrentQueue <KeyValuePair <KeyValuePair <int, string>, CookieContainer> >(); var companyDic = new ConcurrentDictionary <string, int>(); using (var db = new MangningXssDBEntities()) { var companyArr = db.ZhaoPinCompany.Where(w => w.Source.Contains("MANUAL")).Select(s => s.Id).ToArray(); var paramArr = db.ZhaopinStaff.Where(w => companyArr.Any(a => a == w.CompanyId) && !string.IsNullOrEmpty(w.Cookie)).Select(s => new { s.CompanyId, s.Username, s.Cookie }).ToArray(); foreach (var item in paramArr) { companyDic[item.Username] = 0; cookieQueue.Enqueue(new KeyValuePair <KeyValuePair <int, string>, CookieContainer>(new KeyValuePair <int, string>(item.CompanyId, item.Username), item.Cookie.Serialize(".zhaopin.com"))); } } using (var db = new MangningXssDBEntities()) { var list = db.ZhaopinCleaningProcedure.Where(w => w.IsEnable && !w.IsOnline && !string.IsNullOrEmpty(w.Cookie)); foreach (var item in list) { if (item.StartTime < DateTime.Today) { item.TodayWatch = 0; } companyDic[item.Account] = item.TodayWatch; cookieQueue.Enqueue(new KeyValuePair <KeyValuePair <int, string>, CookieContainer>(new KeyValuePair <int, string>(0, item.Account), item.Cookie.Serialize(".zhaopin.com"))); item.IsOnline = true; item.StartTime = DateTime.Now; } db.SaveChanges(); } Console.WriteLine($"已获取 Cookie 数 => {cookieQueue.Count}"); for (var j = 0; j < 16; j++) { Task.Run(() => { try { while (true) { KeyValuePair <KeyValuePair <int, string>, CookieContainer> temp; if (!cookieQueue.TryDequeue(out temp)) { Thread.Sleep(1000); continue; } while (true) { var condition = GetSingleCondition(temp.Key.Value); if (condition == null) { Console.WriteLine($"Company => {temp.Key.Value} 找不到可搜索的条件!"); Thread.Sleep(1000); continue; } var pageIndex = 1; var pageTotal = 1; var total = 0; var isbreak = false; var workYearsEnd = condition.WorkYears == 30 ? 30 : condition.WorkYears + 1; while (pageIndex < pageTotal + 1) { var url = $"https://ihrsearch.zhaopin.com/Home/ResultForCustom?SF_1_1_8={condition.Age}%2C{condition.Age}&SF_1_1_9={condition.Gender}&SF_1_1_4={condition.WorkYears}%2C{workYearsEnd}&SF_1_1_5={condition.Degrees}%2C{condition.Degrees}&orderBy=BIRTH_YEAR%2C0&SF_1_1_27=0&SF_1_1_7={updateRange}%2C9&exclude=1&pageIndex={pageIndex}&pageSize=60"; var requestResult = RequestFactory.QueryRequest(url, cookieContainer: temp.Value, referer: url); if (!requestResult.IsSuccess) { LogFactory.Warn($"Company => {temp.Key.Value} 条件搜索异常!异常原因=>{requestResult.ErrorMsg} Condition=>{JsonConvert.SerializeObject(condition)}"); continue; } if (pageTotal == 1) { var match = Regex.Match(requestResult.Data, "(?s)<span>(\\d+)</span>份简历.+?rd-resumelist-pageNum\">1/(\\d+)</span>"); if (!match.Success) { if (requestResult.Data.Contains("text/javascript\" r='m'")) { LogFactory.Warn($"Cookie 过期,过期用户 => {temp.Key.Value}"); RemoveCookie(temp.Key.Value, companyDic[temp.Key.Value]); } else { LogFactory.Warn($"Company => {temp.Key.Value} 条件搜索异常!返回页面解析异常!{requestResult.Data}"); } isbreak = true; break; } total = Convert.ToInt32(match.Result("$1")); pageTotal = Convert.ToInt32(match.Result("$2")); if (total == 0) { Console.WriteLine("该条件无搜索结果!"); break; } } Console.WriteLine($"Company => {temp.Key.Value} 第 {pageIndex} 页 共 {pageTotal} 页 {total} 个结果"); pageIndex++; //var matchs = Regex.Matches(requestResult.Data, "RedirectToRd/(.+?)','(.+?)','(.+?)',this\\);this"); var matchs = Regex.Matches(requestResult.Data, "(?s)RedirectToRd/([^\r\n]+?)','([^\r\n]+?)','([^\r\n]+?)',this\\);this.+?(\\d{2}-\\d{2}-\\d{2})"); var index = 0; foreach (Match item in matchs) { var number = item.Result("$1").Substring(0, 10); var numberParam = item.Result("$1").Substring(0, item.Result("$1").IndexOf("?", StringComparison.Ordinal)); DateTime updateDateTime; if (DateTime.TryParse(item.Result("$4"), out updateDateTime)) { if (QueryResumeIsExists(number, updateDateTime)) { continue; } } else { if (QueryResumeIsExists(number)) { continue; } } requestResult = RequestFactory.QueryRequest($"http://ihr.zhaopin.com/resumesearch/getresumedetial.do?resumeNo={numberParam}&searchresume=1&resumeSource=1&keyword=%E8%AF%B7%E8%BE%93%E5%85%A5%E7%AE%80%E5%8E%86%E5%85%B3%E9%94%AE%E8%AF%8D%EF%BC%8C%E5%A4%9A%E5%85%B3%E9%94%AE%E8%AF%8D%E5%8F%AF%E7%94%A8%E7%A9%BA%E6%A0%BC%E5%88%86%E9%9A%94&t={item.Result("$2")}&k={item.Result("$3")}&v=undefined&version=3&openFrom=1", cookieContainer: temp.Value); if (!requestResult.IsSuccess) { LogFactory.Warn($"Company => {temp.Key.Value} 简历详情查看异常!异常原因=>{requestResult.ErrorMsg} ResumeNumber=>{number}"); continue; } try { var jsonObj = JsonConvert.DeserializeObject <dynamic>(requestResult.Data); if ((int)jsonObj["code"] != 1) { LogFactory.Warn($"Company => {temp.Key.Value} ResumeNumber => {number} 查看详情异常 信息:{(string)jsonObj["message"]} 查看简历份数:{companyDic[temp.Key.Value]}"); if (((string)jsonObj["message"]).Contains("当日查看简历已达上限")) { RemoveCookie(temp.Key.Value, companyDic[temp.Key.Value]); isbreak = true; break; } continue; } var resumeData = jsonObj.data; SaveCacheByOss(new KeyValuePair <KeyValuePair <int, string>, dynamic>(temp.Key, resumeData)); Console.WriteLine($"Company => {temp.Key.Value} 查看简历成功!查看简历份数:{++companyDic[temp.Key.Value]} ResumeNumber => {number} {++index}/{matchs.Count}"); } catch (Exception ex) { while (true) { if (ex.InnerException == null) { break; } ex = ex.InnerException; } LogFactory.Error($"保存简历信息异常!异常信息:{ex.Message} 响应信息:{requestResult.Data}"); } //if (companyDic[temp.Key.Value] > 1500) //{ // RemoveCookie(temp.Key.Value, companyDic[temp.Key.Value]); // isbreak = true; // break; //} } if (isbreak) { break; } } if (isbreak) { break; } SetSearchStatus(condition.Id, 1, total); } } } catch (Exception ex) { while (true) { if (ex.InnerException == null) { break; } ex = ex.InnerException; } LogFactory.Error($"Watch异常!异常信息:{ex.Message} {ex.StackTrace}"); } }); } }
public void SaveResumeInfo(ResumeMatchResult model) { using (var xdb = new MangningXssDBEntities()) { var user = xdb.ZhaopinUser.FirstOrDefault(f => f.Id == model.UserId); if (user == null) { xdb.ZhaopinUser.Add(new ZhaopinUser { Id = model.UserId, Source = "Watch", ModifyTime = model.ModifyTime, Cellphone = model.Cellphone, Email = model.Email, Name = model.Name, UpdateTime = DateTime.UtcNow, CreateTime = model.CreateTime }); } else { user.ModifyTime = model.ModifyTime; user.UpdateTime = DateTime.UtcNow; user.Name = model.Name; } var resumeEntity = xdb.ZhaopinResume.FirstOrDefault(f => f.Id == model.ResumeId); var userExtId = Regex.IsMatch(model.UserExtId, @"^J[MRSL]\d{9}$") ? model.UserExtId : string.Empty; if (resumeEntity == null) { xdb.ZhaopinResume.Add(new ZhaopinResume { Id = model.ResumeId, RandomNumber = model.ResumeNumber, UserId = model.UserId, RefreshTime = model.ModifyTime, UpdateTime = DateTime.UtcNow, UserExtId = userExtId, Source = "Watch" }); } else { resumeEntity.RandomNumber = model.ResumeNumber; resumeEntity.UserId = model.UserId; resumeEntity.RefreshTime = model.ModifyTime; if (string.IsNullOrEmpty(resumeEntity.UserExtId)) { resumeEntity.UserExtId = userExtId; } } var cache = xdb.ZhaopinMatchedCache.FirstOrDefault(f => f.ResumeId == model.ResumeId); if (cache == null) { xdb.ZhaopinMatchedCache.Add(new ZhaopinMatchedCache { ResumeId = model.ResumeId, ModifyTime = model.ModifyTime, Cellphone = model.Cellphone, Email = model.Email, Name = model.Name, Path = model.Path, ResumeNumber = model.ResumeNumber, UserExtId = model.UserExtId, UserId = model.UserId }); } else { cache.Cellphone = model.Cellphone; cache.Email = model.Email; cache.UserExtId = model.UserExtId; } var watched = xdb.ZhaopinWatchedResume.FirstOrDefault(f => f.Id == model.ResumeId); if (watched == null) { xdb.ZhaopinWatchedResume.Add(new ZhaopinWatchedResume { Id = model.ResumeId, ResumeNumber = model.ResumeNumber, CompanyId = model.CompanyId, WatchTime = DateTime.UtcNow }); } xdb.SaveChanges(); } }
/// <summary> /// 组合简历 /// </summary> private static void PortfolioResume() { while (true) { try { ZhaopinResume resume; var stopwatch = new Stopwatch(); stopwatch.Restart(); if (!resumeQueue.TryDequeue(out resume)) { Thread.Sleep(100); continue; } var filePath = $"{uploadFilePath}{resume.Id}.json"; if (File.Exists(filePath)) { uploadQueue.Enqueue(filePath); continue; } string userId; string cellphone; string email; using (var db = new MangningXssDBEntities()) { var user = db.ZhaopinUser.AsNoTracking().FirstOrDefault(f => f.Id == resume.UserId); if (user == null) { continue; } userId = user.Id.ToString(); cellphone = user.Cellphone; email = user.Email; using (var stream = new MemoryStream()) { if (mangningOssClient.DoesObjectExist(mangningBucketName, $"Zhaopin/Resume/{resume.Id}")) { var bytes = new byte[1024]; int len; var streamContent = mangningOssClient.GetObject(mangningBucketName, $"Zhaopin/Resume/{resume.Id}").Content; while ((len = streamContent.Read(bytes, 0, bytes.Length)) > 0) { stream.Write(bytes, 0, len); } var resumeContent = Encoding.UTF8.GetString(GZip.Decompress(stream.ToArray())); var resumeObj = JsonConvert.DeserializeObject <dynamic>(resumeContent); var resumeDetail = JsonConvert.DeserializeObject(resumeObj.detialJSonStr.ToString()); resumeDetail.DateModified = user.ModifyTime.ToLocalTime(); resumeDetail.DateCreated = user.CreateTime?.ToLocalTime() ?? resumeDetail.DateCreated; resumeDetail.DateLastReleased = resume.RefreshTime.Value.ToLocalTime(); resumeDetail.DateLastViewed = resume.RefreshTime.Value.ToLocalTime(); resumeObj.detialJSonStr = resumeDetail; resumeContent = JsonConvert.SerializeObject(resumeObj); File.WriteAllText(filePath, resumeContent); uploadQueue.Enqueue(filePath); continue; } if (mangningOssClient.DoesObjectExist(mangningBucketName, $"WatchResume/{resume.Id}")) { var bytes = new byte[1024]; int len; var streamContent = mangningOssClient.GetObject(mangningBucketName, $"WatchResume/{resume.Id}").Content; while ((len = streamContent.Read(bytes, 0, bytes.Length)) > 0) { stream.Write(bytes, 0, len); } var resumeContent = Encoding.UTF8.GetString(GZip.Decompress(stream.ToArray())); var resumeObj = JsonConvert.DeserializeObject <dynamic>(resumeContent); resumeObj.userDetials.mobilePhone = user.Cellphone; resumeObj.userDetials.email = user.Email; var resumeDetail = JsonConvert.DeserializeObject(resumeObj.detialJSonStr.ToString()); resumeDetail.DateModified = user.ModifyTime.ToLocalTime(); resumeDetail.DateCreated = user.CreateTime.Value.ToLocalTime(); resumeDetail.DateLastReleased = resume.RefreshTime.Value.ToLocalTime(); resumeDetail.DateLastViewed = resume.RefreshTime.Value.ToLocalTime(); resumeObj.detialJSonStr = resumeDetail; resumeContent = JsonConvert.SerializeObject(resumeObj); using (var jsonStream = new MemoryStream(GZip.Compress(Encoding.UTF8.GetBytes(resumeContent)))) { mangningOssClient.PutObject(mangningBucketName, $"Zhaopin/Resume/{resume.Id}", jsonStream); } File.WriteAllText(filePath, resumeContent); uploadQueue.Enqueue(filePath); continue; } var zhaopinResume = db.ZhaopinResume.FirstOrDefault(f => f.Id == resume.Id); if (zhaopinResume != null) { zhaopinResume.Flag = 0x2; } } db.SaveChanges(); stopwatch.Stop(); } using (var bdcDb = new BadoucaiAliyunDBEntities()) { bdcDb.CoreResumeZhaopin.Add(new CoreResumeZhaopin { Cellphone = cellphone, Email = email, ResumeKey = userId, Type = "ResumeUserId", IsMatched = false }); bdcDb.SaveChanges(); } Console.WriteLine($"{DateTime.Now} > 简历未找到 Josn 源!ResumeId = {resume.Id}, UserId = {userId}, Elapsed = {stopwatch.ElapsedMilliseconds} ms."); } catch (Exception ex) { Trace.TraceError(ex.ToString()); } } }
/// <summary> /// 导入外部Josn /// </summary> private static void ImportResume() { var importCount = 0; while (true) { try { var filePaths = Directory.GetFiles(importFilePath); if (filePaths.Length == 0) { Thread.Sleep(10 * 1000); continue; } Console.WriteLine($"{DateTime.Now} > {filePaths.Length} Need Import ! "); foreach (var filePath in filePaths) { try { var jsonObj = JsonConvert.DeserializeObject <dynamic>(File.ReadAllText(filePath)); var resumeData = jsonObj.data == null ? jsonObj : jsonObj.data; var resumeDetail = JsonConvert.DeserializeObject(resumeData.detialJSonStr.ToString()); var resumeId = resumeData.resumeId != null ? (int)resumeData.resumeId : resumeDetail.ResumeId != null ? (int)resumeDetail.ResumeId : 0; var cellphone = resumeData.userDetials.mobilePhone.ToString(); if (string.IsNullOrEmpty(cellphone)) { var path = importFailPath + Path.GetFileName(filePath); if (File.Exists(path)) { File.Delete(path); } File.Move(filePath, path); continue; } var resumeNumber = ((string)resumeData.resumeNo).Substring(0, 10); var userId = (int)resumeData.userDetials.userMasterId; var refreshTime = BaseFanctory.GetTime((string)resumeDetail.DateLastReleased).ToUniversalTime(); using (var db = new MangningXssDBEntities()) { var resume = db.ZhaopinResume.FirstOrDefault(f => f.Id == resumeId); var isNeedUpload = true; var sourceFlag = -1; var isUpload = false; if (resume != null) { sourceFlag = resume.Flag; } if (!(resume?.RefreshTime != null && resume.RefreshTime.Value.Date >= refreshTime.Date) || resume?.Flag < 8) { if (resume != null) { resume.RandomNumber = resumeNumber; resume.RefreshTime = refreshTime; resume.UpdateTime = DateTime.UtcNow; if (string.IsNullOrEmpty(resume.UserExtId)) { resume.UserExtId = resumeDetail.UserMasterExtId?.ToString(); } if (resumeData.Flag != null && (int)resumeData.Flag < 0) { if (resume.Flag >= 8 || mangningOssClient.DoesObjectExist(mangningBucketName, $"Zhaopin/Resume/{resume.Id}") || mangningOssClient.DoesObjectExist(mangningBucketName, $"WatchResume/{resume.Id}")) { isNeedUpload = false; } } resume.Flag = 0xE; } else { resume = new ZhaopinResume { Id = resumeId, RandomNumber = resumeNumber, UserId = userId, RefreshTime = refreshTime, UpdateTime = DateTime.UtcNow, UserExtId = resumeDetail.UserMasterExtId.ToString(), DeliveryNumber = null, Source = "Import", Flag = 0xE }; db.ZhaopinResume.Add(resume); } var user = db.ZhaopinUser.FirstOrDefault(f => f.Id == userId); if (user != null) { if (!user.Source.Contains("MANUAL")) { user.ModifyTime = BaseFanctory.GetTime((string)resumeDetail.DateModified).ToUniversalTime(); user.CreateTime = resumeData.Flag != null && (int)resumeData.Flag < 0 ? user.CreateTime : BaseFanctory.GetTime((string)resumeDetail.DateCreated).ToUniversalTime(); user.Cellphone = resumeData.userDetials.mobilePhone.ToString(); user.Email = resumeData.userDetials.email.ToString(); user.Name = resumeData.userDetials.userName.ToString(); user.UpdateTime = DateTime.UtcNow; user.Username = resumeData.userDetials.email.ToString(); } } else { user = new ZhaopinUser { Id = userId, Source = "Import", ModifyTime = BaseFanctory.GetTime((string)resumeDetail.DateModified).ToUniversalTime(), CreateTime = BaseFanctory.GetTime((string)resumeDetail.DateCreated).ToUniversalTime(), Cellphone = resumeData.userDetials.mobilePhone.ToString(), Email = resumeData.userDetials.email.ToString(), Name = resumeData.userDetials.userName.ToString(), UpdateTime = DateTime.UtcNow, Username = resumeData.userDetials.email.ToString() }; db.ZhaopinUser.Add(user); } if (isNeedUpload) { isUpload = true; var resumeContent = JsonConvert.SerializeObject(resumeData); using (var jsonStream = new MemoryStream(GZip.Compress(Encoding.UTF8.GetBytes(resumeContent)))) { mangningOssClient.PutObject(mangningBucketName, $"Zhaopin/Resume/{resume.Id}", jsonStream); } } db.SaveChanges(); } File.Delete(filePath); Console.WriteLine($"{DateTime.Now} > Improt success ! ResumeId = {resumeId}, SourceFlag = {sourceFlag}, IsUpload = {isUpload}, Count = {++importCount}."); } } catch (Exception ex) { var path = importFailPath + Path.GetFileName(filePath); if (File.Exists(path)) { File.Delete(path); } File.Move(filePath, path); Trace.WriteLine($"{DateTime.Now} > Import Error Message = {ex.Message}, Path = {path}."); } } } catch (Exception ex) { Trace.TraceError(ex.ToString()); } } }
public void Watch() { var cookieQueue = new ConcurrentQueue <KeyValuePair <KeyValuePair <int, string>, CookieContainer> >(); using (var db = new MangningXssDBEntities()) { var companyArr = db.ZhaoPinCompany.Where(w => w.Source.Contains("MANUAL")).Select(s => s.Id).ToArray(); var paramArr = db.ZhaopinStaff.Where(w => companyArr.Any(a => a == w.CompanyId) && !string.IsNullOrEmpty(w.Cookie)).Select(s => new { s.CompanyId, s.Username, s.Cookie }).ToArray(); foreach (var item in paramArr) { cookieQueue.Enqueue(new KeyValuePair <KeyValuePair <int, string>, CookieContainer>(new KeyValuePair <int, string>(item.CompanyId, item.Username), item.Cookie.Serialize(".zhaopin.com"))); } } using (var db = new MangningXssDBEntities()) { var list = db.ZhaopinCleaningProcedure.Where(w => w.IsEnable && !string.IsNullOrEmpty(w.Cookie)); foreach (var item in list) { cookieQueue.Enqueue(new KeyValuePair <KeyValuePair <int, string>, CookieContainer>(new KeyValuePair <int, string>(0, item.Account), item.Cookie.Serialize(".zhaopin.com"))); } } #region 插入查询条件 for (var degrees = 5; degrees < 16; degrees += 2) { for (var age = 20; age <= 28; age++) { queue.Enqueue(new { Degrees = degrees, Age = age }); } } #endregion Console.WriteLine($"已获取 Cookie 数 => {cookieQueue.Count}"); for (var j = 0; j < 4; j++) { Task.Run(() => { try { while (true) { KeyValuePair <KeyValuePair <int, string>, CookieContainer> temp; if (!cookieQueue.TryDequeue(out temp)) { Thread.Sleep(1000); continue; } while (true) { dynamic condition; if (!queue.TryDequeue(out condition)) { Console.WriteLine($"Company => {temp.Key.Value} 找不到可搜索的条件!"); Thread.Sleep(1000); continue; } var pageIndex = 1; var pageTotal = 1; var total = 0; var isbreak = false; while (pageIndex < pageTotal + 1) { var url = $"https://ihrsearch.zhaopin.com/Home/ResultForCustom?SF_1_1_7=1%2C9&SF_1_1_5={condition.Degrees}%2C{condition.Degrees}&SF_1_1_18=530%3B854%3B531&SF_1_1_8={condition.Age}%2C{condition.Age}&SF_1_1_27=0&orderBy=DATE_MODIFIED,1&pageSize=60&exclude=1"; var requestResult = RequestFactory.QueryRequest(url, cookieContainer: temp.Value, referer: url); if (!requestResult.IsSuccess) { LogFactory.Warn($"Company => {temp.Key.Value} 条件搜索异常!异常原因=>{requestResult.ErrorMsg} Condition=>{JsonConvert.SerializeObject(condition)}"); continue; } if (pageTotal == 1) { var match = Regex.Match(requestResult.Data, "(?s)<span>(\\d+)</span>份简历.+?rd-resumelist-pageNum\">1/(\\d+)</span>"); if (!match.Success) { if (requestResult.Data.Contains("text/javascript\" r='m'")) { LogFactory.Warn($"Cookie 过期,过期用户 => {temp.Key.Value}"); } else { LogFactory.Warn($"Company => {temp.Key.Value} 条件搜索异常!返回页面解析异常!{requestResult.Data}"); } isbreak = true; break; } total = Convert.ToInt32(match.Result("$1")); pageTotal = Convert.ToInt32(match.Result("$2")); if (total == 0) { Console.WriteLine("该条件无搜索结果!"); break; } } Console.WriteLine($"Company => {temp.Key.Value} 第 {pageIndex} 页 共 {pageTotal} 页 {total} 个结果"); pageIndex++; var matchs = Regex.Matches(requestResult.Data, "(?s)RedirectToRd/([^\r\n]+?)','([^\r\n]+?)','([^\r\n]+?)',this\\);this.+?(\\d{2}-\\d{2}-\\d{2})"); using (var db = new MangningXssDBEntities()) { foreach (Match item in matchs) { var number = item.Result("$1").Substring(0, 10); var numberParam = item.Result("$1").Substring(0, item.Result("$1").IndexOf("?", StringComparison.Ordinal)); if (!hashSet.Add(number)) { continue; } db.ZhaopinResumeNumber.Add(new ZhaopinResumeNumber { ResumeNumber = number }); var resume = db.ZhaopinResume.FirstOrDefault(f => f.RandomNumber == number); if (resume == null) { continue; } var user = db.ZhaopinUser.FirstOrDefault(f => f.Id == resume.UserId && !string.IsNullOrEmpty(f.Cellphone)); if (user == null) { continue; } requestResult = RequestFactory.QueryRequest($"http://ihr.zhaopin.com/resumesearch/getresumedetial.do?resumeNo={numberParam}&searchresume=1&resumeSource=1&keyword=%E8%AF%B7%E8%BE%93%E5%85%A5%E7%AE%80%E5%8E%86%E5%85%B3%E9%94%AE%E8%AF%8D%EF%BC%8C%E5%A4%9A%E5%85%B3%E9%94%AE%E8%AF%8D%E5%8F%AF%E7%94%A8%E7%A9%BA%E6%A0%BC%E5%88%86%E9%9A%94&t={item.Result("$2")}&k={item.Result("$3")}&v=undefined&version=3&openFrom=1", cookieContainer: temp.Value); if (!requestResult.IsSuccess) { LogFactory.Warn($"Company => {temp.Key.Value} 简历详情查看异常!异常原因=>{requestResult.ErrorMsg} ResumeNumber=>{number}"); continue; } try { var jsonObj = JsonConvert.DeserializeObject <dynamic>(requestResult.Data); if ((int)jsonObj["code"] != 1) { LogFactory.Warn($"Company => {temp.Key.Value} ResumeNumber => {number} 查看详情异常 信息:{(string)jsonObj["message"]}"); if (((string)jsonObj["message"]).Contains("当日查看简历已达上限")) { isbreak = true; break; } continue; } var resumeData = jsonObj.data; resumeData.userDetials.mobilePhone = user.Cellphone; resumeData.userDetials.email = user.Email; File.WriteAllText($@"D:\ResumeBusiness\{number}.json", JsonConvert.SerializeObject(resumeData)); Console.WriteLine($"Company => {temp.Key.Value} 查看简历成功!ResumeNumber => {number}"); } catch (Exception ex) { while (true) { if (ex.InnerException == null) { break; } ex = ex.InnerException; } LogFactory.Error($"保存简历信息异常!异常信息:{ex.Message} 响应信息:{requestResult.Data}"); } } db.SaveChanges(); } if (isbreak) { break; } } if (isbreak) { break; } } } } catch (Exception ex) { while (true) { if (ex.InnerException == null) { break; } ex = ex.InnerException; } LogFactory.Error($"Watch异常!异常信息:{ex.Message} {ex.StackTrace}"); } }); } }