/// <summary> /// 刷新简历下载数 /// </summary> /// <param name="email"></param> /// <param name="cookie"></param> /// <param name="host"></param> private static void RefreshFreeDownloadNumber(string email, CookieContainer cookie, string host) { var dataResult = RequestFactory.QueryRequest("http://www.jianlika.com/Search", cookieContainer: cookie, host: host); if (!dataResult.IsSuccess) { LogFactory.Warn($"用户登录刷新下载数异常!异常用户:{email}", MessageSubjectEnum.JianLiKa); return; } var html = dataResult.Data; if (Regex.IsMatch(html, "class=\"ico-png-money\"></i><span>(\\d+)")) { var count = Convert.ToInt32(Regex.Match(html, "class=\"ico-png-money\"></i><span>(\\d+)").Result("$1")); using (var db = new ResumeMatchDBEntities()) { var user = db.User.FirstOrDefault(f => f.Email == email); if (user == null) { return; } user.DownloadNumber = count; db.TransactionSaveChanges(); } } }
//private static ActionBlock<KeyValuePair<string,List<ResumeSearch>>> actionBlock = new ActionBlock<KeyValuePair<string, List<ResumeSearch>>>(data => //{ // Work(data.Key, data.Value); //}, new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = 1 }); /// <summary> /// 工作流 /// </summary> /// <param name="host"></param> /// <param name="resumeSearches"></param> /// <returns></returns> private static bool Work(string host, IEnumerable <ResumeSearch> resumeSearches) { //foreach (var platform in Enum.GetValues(typeof(MatchPlatform))) var isUnBreak = true; using (var db = new ResumeMatchDBEntities()) { if (resumeSearches.Any(resume => !MatchResume(resume, host))) { isUnBreak = false; } var users = db.User.Where(w => w.Host == host).ToList(); foreach (var user in users) { user.IsLocked = false; } db.TransactionSaveChanges(); } return(isUnBreak); }
public static List <ResumeSearch> PullResumes() { List <ResumeSearch> list; lock (lockObj) { while (resumeQueue.IsEmpty || !resumeQueue.TryDequeue(out list)) { if (resumeQueue.IsEmpty) { List <ResumeSearch> resumeList; if (isFirst) { var time = DateTime.UtcNow.AddHours(-1); using (var db = new ResumeMatchDBEntities()) { resumeList = db.ResumeComplete.Where(w => w.Status == 1 && (!w.IsLocked || w.IsLocked && w.LockedTime < time)).Select(s => new ResumeSearch { Degree = s.Degree, Gender = s.Gender, Introduction = s.Introduction, LastCompany = s.LastCompany, Name = s.Name, ResumeId = s.ResumeId, ResumeNumber = s.ResumeNumber, University = s.University, UserMasterExtId = s.UserMasterExtId }).ToList(); } isFirst = false; } else { resumeList = PullAllResumes(); } const int taskCount = 10; for (var i = 0; i < resumeList.Count + taskCount; i += taskCount) { var temp = resumeList.Skip(i).Take(taskCount).ToList(); if (temp.Any()) { resumeQueue.Enqueue(temp); } } } } } return(list); }
public static DataResult <string> GetResumeId(ResumeSearch data, string host) { var dataResult = new DataResult <string>(); CookieContainer cookie; var result = GetUser(userDictionary, host, true, MatchPlatform.FenJianLi, Login, out cookie); if (!result.IsSuccess) { dataResult.IsSuccess = false; dataResult.Code = result.Code; dataResult.ErrorMsg = result.ErrorMsg; return(dataResult); } var user = result.Data; dataResult = GetResumeId(data, cookie, user); using (var db = new ResumeMatchDBEntities()) { var resume = db.ResumeComplete.FirstOrDefault(f => f.ResumeId == data.ResumeId); if (resume != null) { resume.Host = host; resume.MatchPlatform = (short)MatchPlatform.FenJianLi; resume.MatchTime = DateTime.UtcNow; resume.UserId = user.Id; if (!string.IsNullOrWhiteSpace(dataResult.Data)) { resume.Status = 2; resume.MatchResumeId = dataResult.Data; } db.TransactionSaveChanges(); } } return(dataResult); }
/// <summary> /// 解锁Host用户 /// </summary> private static void HostUnLock() { lock (lockObj) { using (var db = new ResumeMatchDBEntities()) { db.UsingTransaction(() => { string host; while (hostQueue.TryDequeue(out host)) { var hostUnLock = host; var users = db.User.Where(w => w.Host == hostUnLock && w.IsLocked).ToList(); foreach (var user in users) { user.IsLocked = false; } } foreach (var item in hostList) { if (string.IsNullOrWhiteSpace(item)) { continue; } var users = db.User.Where(w => w.Host == item && w.IsLocked).ToList(); foreach (var user in users) { user.IsLocked = false; } } db.SaveChanges(); hostList.Clear(); }); } } }
/// <summary> /// 刷新简历下载数 /// </summary> /// <param name="email"></param> /// <param name="cookie"></param> /// <param name="host"></param> private static void RefreshFreeDownloadNumber(string email, CookieContainer cookie, string host) { var userToken = cookie.GetCookies(new Uri("http://qiye.zhaopingou.com/"))["hrkeepToken"]; var param = $"isAjax=1&clientNo=&userToken={userToken?.Value}&clientType=2"; var dataResult = RequestFactory.QueryRequest("http://qiye.zhaopingou.com/zhaopingou_interface/user_information?timestamp=" + BaseFanctory.GetUnixTimestamp(), param, RequestEnum.POST, cookie, host: host); if (!dataResult.IsSuccess) { LogFactory.Warn($"用户登录刷新下载数异常!异常用户:{email}", MessageSubjectEnum.ZhaoPinGou); return; } var jObject = JsonConvert.DeserializeObject(dataResult.Data) as JObject; if (jObject != null) { if ((int)jObject["errorCode"] != 1) { LogFactory.Warn($"用户登录异常!异常用户:{email},异常信息:{(string)jObject["message"]}", MessageSubjectEnum.ZhaoPinGou); return; } using (var db = new ResumeMatchDBEntities()) { var user = db.User.FirstOrDefault(f => f.Email == email); if (user == null) { return; } var count = (int)jObject["memberEvents"]?["free_count"]; user.DownloadNumber = count; db.TransactionSaveChanges(); } } }
public void ChangeResumeStatus(string resumeId, bool isMatchd) { using (var db = new ResumeMatchDBEntities()) { var resume = db.OldResumeSummary.FirstOrDefault(f => f.ResumeId == resumeId); if (resume == null) { return; } resume.Status = 1; resume.MatchTime = DateTime.Now; resume.IsMatched = isMatchd; db.SaveChanges(); } }
/// <summary> /// 工作流 /// </summary> /// <param name="host"></param> /// <param name="resumes"></param> /// <returns></returns> private static void Work(string host, IReadOnlyCollection <ResumeComplete> resumes) { using (var db = new ResumeMatchDBEntities()) { if (resumes.Any(resume => !DownloadResume(host, resume))) { return; } var resumeIdArr = resumes.Select(s => s.Id).ToArray(); var resumeList = db.ResumeComplete.Where(w => resumeIdArr.Any(a => a == w.Id) && w.IsLocked).ToList(); foreach (var item in resumeList) { item.IsLocked = false; } db.TransactionSaveChanges(); } }
/// <summary> /// 解锁简历 /// </summary> private static void ResumeUnLock() { lock (lockObj) { using (var db = new ResumeMatchDBEntities()) { db.UsingTransaction(() => { KeyValuePair <string, List <ResumeComplete> > hostResume; while (resumeQueue.TryDequeue(out hostResume)) { var resumeIdArr = hostResume.Value.Select(s => s.Id).ToArray(); var resumes = db.ResumeComplete.Where(w => resumeIdArr.Any(a => a == w.Id) && w.IsLocked).ToList(); foreach (var resume in resumes) { resume.IsLocked = false; } } foreach (var item in hostResumeList) { var resumeIdArr = item.Value.Select(s => s.Id).ToArray(); var resumes = db.ResumeComplete.Where(w => resumeIdArr.Any(a => a == w.Id) && w.IsLocked).ToList(); foreach (var resume in resumes) { resume.IsLocked = false; } } db.SaveChanges(); }); } } }
public static void HandlerRoutine(int CtrlType) { using (var db = new ResumeMatchDBEntities()) { var users = db.User.Where(w => w.IsLocked).ToList(); foreach (var user in users) { user.IsLocked = false; } var resumes = db.ResumeComplete.Where(w => w.IsLocked).ToList(); foreach (var resume in resumes) { resume.IsLocked = false; } db.TransactionSaveChanges(); } ReleaseProxy(); }
public static DataResult DownloadResume(ResumeComplete data, string host) { var dataResult = new DataResult <string>(); dataResult.IsSuccess = false; CookieContainer cookie; Next: var result = GetUser(userDictionary, host, false, MatchPlatform.FenJianLi, Login, out cookie); if (!result.IsSuccess) { dataResult.IsSuccess = false; dataResult.Code = result.Code; dataResult.ErrorMsg = result.ErrorMsg; return(dataResult); } var user = result.Data; if (string.IsNullOrWhiteSpace(user.FolderCode)) { dataResult = GetFolderId(cookie, host); if (!dataResult.IsSuccess) { return(dataResult); } user.FolderCode = dataResult.Data; } var unLockResult = UnLockResume(cookie, user, data.MatchResumeId, host, user.FolderCode); using (var db = new ResumeMatchDBEntities()) { var resumeEntity = db.ResumeComplete.FirstOrDefault(f => f.Id == data.Id); if (resumeEntity == null) { LogFactory.Warn($"找不到简历!Id:{data.Id}", MessageSubjectEnum.FenJianLi); return(dataResult); } var userEntity = db.User.FirstOrDefault(f => f.Id == user.Id); if (userEntity == null) { LogFactory.Warn($"找不到用户!用户:{user.Email}", MessageSubjectEnum.FenJianLi); db.TransactionSaveChanges(); return(dataResult); } userEntity.FolderCode = user.FolderCode; if (!unLockResult.IsSuccess) { resumeEntity.Status = 5; if (unLockResult.Code == ResultCodeEnum.NoDownloadNumber) { user.DownloadNumber = 0; userEntity.DownloadNumber = 0; resumeEntity.Status = 2; user.LastLoginTime = DateTime.UtcNow; userEntity.LastLoginTime = DateTime.UtcNow; db.TransactionSaveChanges(); goto Next; } db.TransactionSaveChanges(); return(unLockResult); } resumeEntity.DownloadTime = DateTime.UtcNow; resumeEntity.Status = 4; resumeEntity.UserId = user.Id; user.DownloadNumber--; if (user.DownloadNumber == 0) { userEntity.LastLoginTime = DateTime.UtcNow; } userEntity.DownloadNumber = user.DownloadNumber; var id = data.MatchResumeId.Substring(0, data.MatchResumeId.IndexOf("/", StringComparison.Ordinal)); dataResult = ResumeDetailSpider(id, cookie, host); if (!dataResult.IsSuccess) { resumeEntity.Status = 7; dataResult.ErrorMsg = "获取简历详情异常!"; dataResult.IsSuccess = false; db.TransactionSaveChanges(); return(dataResult); } var jObject = JsonConvert.DeserializeObject(dataResult.Data) as JObject; if (jObject == null) { resumeEntity.Status = 7; dataResult.ErrorMsg = "简历反序列化异常!"; dataResult.IsSuccess = false; SaveFile(id, dataResult.Data, user.Email); db.TransactionSaveChanges(); return(dataResult); } var email = (string)jObject["contact"]?["email"]; var cellphone = (string)jObject["contact"]?["mobile"]; if (cellphone == null) { resumeEntity.Status = 7; LogFactory.Warn($"补全简历异常!电话为空,ResumeId:{data.ResumeId}", MessageSubjectEnum.FenJianLi); SaveFile(id, dataResult.Data, user.Email); db.TransactionSaveChanges(); dataResult.IsSuccess = false; return(dataResult); } resumeEntity.PostBackStatus = 2; resumeEntity.Status = 6; var matchedResult = new List <ResumeMatchResult> { new ResumeMatchResult { ResumeNumber = data.ResumeNumber, Cellphone = cellphone, Email = email, Status = 2 } }; SaveFile(id, dataResult.Data, user.Email); if (ApiBase.PostResumes(matchedResult)) { resumeEntity.PostBackStatus = 1; } resumeEntity.Email = email; resumeEntity.Cellphone = cellphone; db.TransactionSaveChanges(); dataResult.IsSuccess = true; } return(dataResult); }
/// <summary> /// 匹配简历 /// </summary> /// <param name="data"></param> /// <param name="host"></param> /// <returns></returns> private static bool MatchResume(ResumeSearch data, string host) { var dataResult = Platform.ZhaoPinGou.SearchResumeSpider.GetResumeId(data, host); if (dataResult == null) { return(false); } if (!dataResult.IsSuccess) { switch (dataResult.Code) { case ResultCodeEnum.ProxyDisable: LogFactory.Info($"Host:{host} 代理失效!", MessageSubjectEnum.ZhaoPinGou); return(string.IsNullOrWhiteSpace(host) || host == "210.83.225.31:15839"); case ResultCodeEnum.RequestUpperLimit: LogFactory.Info($"Host:{host} 请求达到当日上限!", MessageSubjectEnum.ZhaoPinGou); return(string.IsNullOrWhiteSpace(host) || host == "210.83.225.31:15839"); case ResultCodeEnum.NoUsers: LogFactory.Info($"Host:{host} 对应的Host没有可用用户!", MessageSubjectEnum.ZhaoPinGou); return(string.IsNullOrWhiteSpace(host) || host == "210.83.225.31:15839"); case ResultCodeEnum.WebNoConnection: LogFactory.Warn("网站无法建立链接!", MessageSubjectEnum.ZhaoPinGou); break; default: LogFactory.Warn($"匹配结果返回异常!异常消息:{dataResult.ErrorMsg} ", MessageSubjectEnum.ZhaoPinGou); return(string.IsNullOrWhiteSpace(host) || host == "210.83.225.31:15839"); } } if (!string.IsNullOrWhiteSpace(dataResult.Data)) { LogFactory.Info($"匹配成功!简历ID:{data.ResumeId},姓名:{data.Name}", MessageSubjectEnum.ZhaoPinGou); return(true); } dataResult = Platform.FenJianLi.SearchResumeSpider.GetResumeId(data, host); if (dataResult == null) { return(false); } if (!dataResult.IsSuccess) { switch (dataResult.Code) { case ResultCodeEnum.ProxyDisable: LogFactory.Info($"Host:{host} 代理失效!", MessageSubjectEnum.FenJianLi); return(string.IsNullOrWhiteSpace(host) || host == "210.83.225.31:15839"); case ResultCodeEnum.RequestUpperLimit: LogFactory.Info($"Host:{host} 请求达到当日上限!", MessageSubjectEnum.FenJianLi); return(string.IsNullOrWhiteSpace(host) || host == "210.83.225.31:15839"); case ResultCodeEnum.NoUsers: LogFactory.Info($"Host:{host} 对应的Host没有可用用户!", MessageSubjectEnum.FenJianLi); return(string.IsNullOrWhiteSpace(host) || host == "210.83.225.31:15839"); case ResultCodeEnum.WebNoConnection: LogFactory.Warn("网站无法建立链接!", MessageSubjectEnum.FenJianLi); break; default: LogFactory.Warn($"匹配结果返回异常!异常消息:{dataResult.ErrorMsg} ", MessageSubjectEnum.FenJianLi); return(string.IsNullOrWhiteSpace(host) || host == "210.83.225.31:15839"); } } if (!string.IsNullOrWhiteSpace(dataResult.Data)) { LogFactory.Info($"匹配成功!简历ID:{data.ResumeId},姓名:{data.Name}", MessageSubjectEnum.FenJianLi); return(true); } using (var db = new ResumeMatchDBEntities()) { var resume = db.ResumeComplete.FirstOrDefault(f => f.ResumeId == data.ResumeId); if (resume != null) { resume.Status = 3; if (dataResult.Code == ResultCodeEnum.WebNoConnection) { resume.Status = 9; } LogFactory.Info($"匹配失败!简历ID:{resume.ResumeId},姓名:{resume.Name}"); db.TransactionSaveChanges(); } } return(true); }
public void Search() { var resumeQueue = new ConcurrentQueue <OldResumeSummary>(); Task.Run(() => { var pageIndex = 0; while (true) { if (resumeQueue.Count < 100) { try { using (var db = new ResumeMatchDBEntities()) { var resumeList = db.OldResumeSummary.OrderBy(o => o.Id).Skip(pageIndex * 5000).Take(5000).ToList(); resumeList.ForEach(f => { resumeQueue.Enqueue(f); }); } pageIndex++; } catch (Exception ex) { while (true) { if (ex.InnerException == null) { break; } ex = ex.InnerException; } LogFactory.Error(ex.Message); } } Thread.Sleep(TimeSpan.FromSeconds(1)); } }); var queue = new ConcurrentQueue <string>(); var sjhArr = "130,131,132,155,156,185,186,145,171,1707,1708,1709,166,146,1349,173,133,153,177,180,181,189,149,1700,1701,1702,199".Split(","); for (var j = 0; j < 32; j++) { Task.Run(() => { while (true) { OldResumeSummary resume; if (!resumeQueue.TryDequeue(out resume)) { continue; } var cellphoneStart = resume.Cellphone.Substring(0, 3); if (!sjhArr.Contains(cellphoneStart)) { continue; } var filePath = $@"E:\智联招聘\{resume.Template}\{resume.ResumeId}.{Path.GetFileNameWithoutExtension(resume.Template)}"; if (!File.Exists(filePath)) { LogFactory.Warn($"指定路径不存在!ResumeNumber=>{resume.ResumeId} Path=>{filePath}"); continue; } var sourceCode = File.ReadAllText(filePath); var genderMatch = Regex.Match(sourceCode, "(男|女)"); if (!genderMatch.Success || genderMatch.Value == "男") { continue; } var addressMatch = Regex.Match(sourceCode, "(广州|北京|上海)"); if (addressMatch.Success) { continue; } queue.Enqueue(resume.Cellphone); } }); } var ltsb = new StringBuilder(); var dxsb = new StringBuilder(); Task.Run(() => { var ltcount = 0; var dxcount = 0; const string liantong = @"D:\360安全浏览器下载\联通NEW.csv"; const string dianxin = @"D:\360安全浏览器下载\电信NEW.csv"; var ltArr = File.ReadAllLines(liantong).ToList(); var dxArr = File.ReadAllLines(dianxin).ToList(); var arr = new List <string>(); arr.AddRange(ltArr.Select(s => s.Substring(0, 11))); arr.AddRange(dxArr.Select(s => s.Substring(0, 11))); while (true) { string cellphone; if (!queue.TryDequeue(out cellphone)) { continue; } var yysString = string.Empty; var cellphoneStart = cellphone.Substring(0, 3); if ("130,131,132,155,156,185,186,145,176".Split(",").Contains(cellphoneStart)) { yysString = "联通"; } else if ("133,153,177,180,181,189".Split(",").Contains(cellphoneStart)) { yysString = "电信"; } if (string.IsNullOrEmpty(yysString) || arr.Contains(cellphone)) { continue; } if (yysString == "电信") { dxsb.AppendLine(cellphone); if (++dxcount % 10000 == 0) { File.WriteAllText($@"D:\360安全浏览器下载\手机号数据\旧库{yysString}_{dxcount / 10000}数据.txt", dxsb.ToString()); dxsb.Clear(); } } else { ltsb.AppendLine(cellphone); if (++ltcount % 10000 == 0) { File.WriteAllText($@"D:\360安全浏览器下载\手机号数据\旧库{yysString}_{ltcount / 10000}数据.txt", ltsb.ToString()); ltsb.Clear(); } } } }); SpinWait.SpinUntil(() => false); }
/// <summary> /// 搜索旧库 /// </summary> /// <param name="list"></param> /// <returns></returns> public static List <ResumeMatchResult> SearchOldLibrary(IEnumerable <dynamic> list) { var resumeList = new List <ResumeMatchResult>(); using (var db = new ResumeMatchDBEntities()) { foreach (var item in list) { var resumeNumber = ((string)item.ResumeNumber).Substring(0, 10); var resume = db.OldResumeSummary.FirstOrDefault(w => w.ResumeId == resumeNumber); if (resume == null) { continue; } resume.MatchTime = DateTime.Now; resume.IsMatched = true; resumeList.Add(new ResumeMatchResult { Cellphone = resume.Cellphone, Email = resume.Email, ResumeNumber = item.ResumeNumber, Status = 2 }); using (var xdb = new MangningXssDBEntities()) { var filePath = "X:" + ((string)item.SavePath).Replace("/", "\\"); if (!File.Exists(filePath)) { LogFactory.Warn($"找不到文件路径:{filePath}, ResumeNumber:{resumeNumber}"); continue; } var resumeJson = JsonConvert.DeserializeObject <dynamic>(File.ReadAllText(filePath)); var detail = JsonConvert.DeserializeObject(resumeJson.detialJSonStr.ToString()); var resumeId = resumeJson.resumeId != null ? (int)resumeJson.resumeId : detail.ResumeId != null ? (int)detail.ResumeId : 0; if (resumeId == 0) { LogFactory.Warn($"解析异常!ResumeId 为空, ResumeNumber:{resumeNumber}"); continue; } var userId = (int)resumeJson.userDetials.userMasterId; var user = xdb.ZhaopinUser.FirstOrDefault(f => f.Id == userId); if (user != null) { if (!user.Source.Contains("MANUAL")) { user.Id = userId; user.Source = "XSS"; user.ModifyTime = BaseFanctory.GetTime((string)detail.DateModified).ToUniversalTime(); user.CreateTime = BaseFanctory.GetTime((string)detail.DateCreated).ToUniversalTime(); user.Cellphone = resume.Cellphone; user.Email = resume.Email; user.Name = resumeJson.userDetials.userName.ToString(); user.UpdateTime = DateTime.UtcNow; user.Username = resumeJson.userDetials.email.ToString(); } } else { xdb.ZhaopinUser.AddOrUpdate(a => a.Id, new ZhaopinUser { Id = userId, Source = "XSS", ModifyTime = BaseFanctory.GetTime((string)detail.DateModified).ToUniversalTime(), CreateTime = BaseFanctory.GetTime((string)detail.DateCreated).ToUniversalTime(), Cellphone = resume.Cellphone, Email = resume.Email, Name = resumeJson.userDetials.userName.ToString(), UpdateTime = DateTime.UtcNow, Username = resumeJson.userDetials.email.ToString() }); } var resumeEntity = xdb.ZhaopinResume.FirstOrDefault(f => f.Id == resumeId); if (resumeEntity == null) { xdb.ZhaopinResume.Add(new ZhaopinResume { Id = resumeId, RandomNumber = resumeNumber, UserId = userId, RefreshTime = BaseFanctory.GetTime((string)detail.DateModified).ToUniversalTime(), UpdateTime = DateTime.UtcNow, UserExtId = detail.UserMasterExtId.ToString(), DeliveryNumber = null, Source = "XSS" }); } else { resumeEntity.Id = resumeId; resumeEntity.RandomNumber = resumeNumber; resumeEntity.UserId = userId; resumeEntity.RefreshTime = BaseFanctory.GetTime((string)detail.DateModified).ToUniversalTime(); resumeEntity.UpdateTime = DateTime.UtcNow; resumeEntity.UserExtId = detail.UserMasterExtId.ToString(); resumeEntity.DeliveryNumber = resumeEntity.DeliveryNumber; resumeEntity.Source = resumeEntity.Source; } xdb.SaveChanges(); var path = $"{ConfigurationManager.AppSettings["Resume.SavePath"]}{DateTime.Now:yyyyMMdd}"; if (!Directory.Exists(path)) { Directory.CreateDirectory(path); } var resumePath = $@"{path}\{resumeId}.json"; File.WriteAllText(resumePath, JsonConvert.SerializeObject(resumeJson)); uploadOssActionBlock.Post(resumePath); } } db.TransactionSaveChanges(); } return(resumeList); }
/// <summary> /// 匹配 /// </summary> private void Match() { lock (lockObj) { if (!hostQueue.Any() && isFirst) { hostQueue.Enqueue("210.83.225.31:15839"); isFirst = false; } var users = new List <User>(); using (var db = new ResumeMatchDBEntities()) { db.UsingTransaction(() => { var dateTime = DateTime.UtcNow.AddHours(-1); var nowDate = DateTime.UtcNow.Date; users = db.User .Where(w => w.IsEnable && w.Status == 1 && (w.RequestDate.Value == null || w.RequestDate.Value < nowDate || w.RequestDate.Value == nowDate && w.RequestNumber < Global.TodayMaxRequestNumber) && (!w.IsLocked || w.IsLocked && w.LockedTime < dateTime) && (string.IsNullOrEmpty(w.Host) || !string.IsNullOrEmpty(w.Host) == Global.IsEnanbleProxy)) .OrderBy(o => o.RequestNumber) .ThenByDescending(o => o.Host) //.Take(Global.PlatformCount * Global.PlatformHostCount * 2) .ToList(); foreach (var user in users) { user.IsLocked = true; user.LockedTime = DateTime.UtcNow; } db.SaveChanges(); }); } var hosts = users .Where(w => w.IsEnable && w.Host != "210.83.225.31:15839") .GroupBy(g => g.Host) .Select(s => new { Host = s.Key, Count = s.Count() }) .OrderByDescending(o => o.Count) .ToList(); hosts.ForEach(f => { hostQueue.Enqueue(f.Host); }); } string host; while (hostQueue.TryDequeue(out host)) { var hostTemp = host; hostList.Add(hostTemp); if (!string.IsNullOrWhiteSpace(hostTemp)) { GetProxy("Match", hostTemp); } while (true) { List <ResumeSearch> resumes; while (true) { try { resumes = ResumeProducer.PullResumes(); break; } catch (Exception ex) { while (true) { if (ex.InnerException == null) { break; } ex = ex.InnerException; } LogFactory.Error($"拉取待匹配的简历异常!异常信息:{ex.Message},堆栈信息:{ex.StackTrace}"); } } if (resumes != null && resumes.Count > 0) { Console.WriteLine(string.Join(",", resumes.Select(s => s.Name).ToArray())); if (!Work(hostTemp, resumes)) { break; } } } ReleaseProxy("Match", hostTemp); //hostList.Remove(hostTemp); LogFactory.Info("Host 消费记录:" + JsonConvert.SerializeObject(hostQueue)); } HostUnLock(); }
/// <summary> /// 过滤器 /// </summary> /// <param name="list"></param> /// <returns></returns> private static List <ResumeSearch> FilterExist(List <ResumeSearch> list) { Login: lock (lockObj) { if (string.IsNullOrWhiteSpace(signature)) { signature = Login(); } } using (var db = new ResumeMatchDBEntities()) { var resumeIdArr = list.Select(s => s.ResumeId).ToArray(); var resumeList = db.ResumeComplete.Where(w => resumeIdArr.Any(a => a == w.ResumeId)).ToList(); foreach (var resume in resumeList) { resume.Status = 1; resume.LibraryExist = 3; } if (string.IsNullOrWhiteSpace(signature)) { db.TransactionSaveChanges(); LogFactory.Warn("简历过滤 API 登录异常!跳过过滤!", MessageSubjectEnum.API); return(list); } dynamic param = new { Username = Global.UserName, Signature = signature, ResumeSummaries = new List <object>() }; param.ResumeSummaries.AddRange(list.Select(s => new { s.ResumeNumber, s.ResumeId, UserMasterExtendId = s.UserMasterExtId }).ToList()); for (var i = 0; i < 3; i++) { var dataResult = RequestFactory.QueryRequest(Global.HostChen + "/api/queryresume/query", JsonConvert.SerializeObject(param), RequestEnum.POST, contentType: ContentTypeEnum.Json.Description()); if (dataResult.IsSuccess) { var jObject = JsonConvert.DeserializeObject(dataResult.Data) as JObject; if (jObject == null) { continue; } if ((int)jObject["Code"] == 3) { signature = string.Empty; goto Login; } if ((int)jObject["Code"] == 0) { var jArray = jObject["ResumeSummaries"] as JArray; if (jArray == null) { continue; } if (jArray.Count > 0) { Global.TotalMatchSuccess += jArray.Count; Global.TotalDownload += jArray.Count; var matchedResult = jArray.Select(s => new ResumeMatchResult { Cellphone = (string)s["Cellphone"], Email = (string)s["Email"], ResumeNumber = (string)s["ResumeNumber"], Status = 2 }).ToList(); var isPostSuccess = PostResumes(matchedResult); var arr = matchedResult.Select(s => s.ResumeNumber).ToArray(); var resumes = db.ResumeComplete.Where(w => arr.Any(a => a.Equals(w.ResumeNumber))).ToList(); foreach (var resume in resumes) { resume.Status = 6; resume.Cellphone = matchedResult.FirstOrDefault(f => f.ResumeNumber == resume.ResumeNumber)?.Cellphone; resume.Email = matchedResult.FirstOrDefault(f => f.ResumeNumber == resume.ResumeNumber)?.Email; resume.DownloadTime = DateTime.UtcNow; resume.LibraryExist = 1; resume.PostBackStatus = isPostSuccess ? (short)1 : (short)2; } list.RemoveAll(r => arr.Any(a => a == r.ResumeNumber)); } var resumesArr = list.Select(s => s.ResumeNumber).ToArray(); var resumeItems = db.ResumeComplete.Where(w => resumesArr.Any(a => a == w.ResumeNumber)).ToList(); foreach (var resume in resumeItems) { resume.Status = 1; resume.LibraryExist = 2; } db.TransactionSaveChanges(); break; } LogFactory.Warn("简历过滤 API 筛选简历异常!异常信息:" + jObject["Message"], MessageSubjectEnum.API); } } } return(list); }
private void Download() { lock (lockObj) { var resumes = new List <ResumeComplete>(); using (var db = new ResumeMatchDBEntities()) { db.UsingTransaction(() => { var dateTime = DateTime.UtcNow.AddHours(-1); var today = DateTime.UtcNow.Date.AddHours(-8); var downloadUserArr = db.User .Where(w => w.IsEnable && (string.IsNullOrEmpty(w.Host) || !string.IsNullOrEmpty(w.Host) == Global.IsEnanbleProxy) && (w.DownloadNumber > 0 || w.LastLoginTime < today || w.LastLoginTime == null)) .GroupBy(g => new { g.Host, g.Platform }) .Select(s => s.Key) .Distinct() .ToList(); var query = db.ResumeComplete .Where(w => w.Status == 2 && (!w.IsLocked || w.IsLocked && w.LockedTime < dateTime)); var platformList = downloadUserArr.GroupBy(g => g.Platform).Select(s => s.Key).ToList(); var hostList = downloadUserArr.GroupBy(g => g.Host) /*.Where(w=>!string.IsNullOrEmpty(w.Key))*/.Select(s => s.Key).ToList(); // 排除本地HOST foreach (var item in platformList) { resumes.AddRange(query .Where(w => w.MatchPlatform == item) .OrderByDescending(o => o.Weights) .ThenByDescending(o => o.MatchTime) .Take(20)); } resumes = resumes.Where(w => w.Weights == 1 && w.MatchPlatform == 4) // Todo:当前只优先下载泽林的简历 .OrderByDescending(o => o.Weights) .ThenByDescending(o => o.MatchTime) .Take(20).ToList(); foreach (var resume in resumes) { resume.IsLocked = true; resume.LockedTime = DateTime.UtcNow; } db.SaveChanges(); var count = resumes.Count / hostList.Count + 1; for (var i = 0; i < hostList.Count; i++) { var temp = resumes.Skip(i * count).Take(count).ToList(); if (temp.Any()) { resumeQueue.Enqueue(new KeyValuePair <string, List <ResumeComplete> >(hostList[i], temp)); } } }); } } KeyValuePair <string, List <ResumeComplete> > hostResume; while (resumeQueue.TryDequeue(out hostResume)) { var hostResumeTemp = hostResume; hostResumeList.Add(hostResumeTemp); if (!string.IsNullOrWhiteSpace(hostResumeTemp.Key)) { GetProxy("Download", hostResumeTemp.Key); } Work(hostResumeTemp.Key, hostResumeTemp.Value); ReleaseProxy("Download", hostResumeTemp.Key); if (isEnd) { ResumeUnLock(); } hostResumeList.Remove(hostResumeTemp); } }
public static DataResult DownloadResume(ResumeComplete data, string host) { if (Filer(data)) { return(new DataResult()); } var dataResult = new DataResult <string>(); dataResult.IsSuccess = false; CookieContainer cookie; Next: var result = GetUser(userDictionary, host, false, MatchPlatform.ZhaoPinGou, Login, out cookie); if (!result.IsSuccess) { dataResult.IsSuccess = false; dataResult.Code = result.Code; dataResult.ErrorMsg = result.ErrorMsg; return(dataResult); } var user = result.Data; var userToken = cookie.GetCookies(new Uri("http://qiye.zhaopingou.com/"))["hrkeepToken"]; if (string.IsNullOrWhiteSpace(user.FolderCode)) { dataResult = GetFolderId(cookie, user, userToken?.Value, data.MatchResumeId, host); if (!dataResult.IsSuccess) { return(dataResult); } user.FolderCode = dataResult.Data; } var unLockResult = UnLockResume(cookie, user, userToken?.Value, data.MatchResumeId, host, user.FolderCode); using (var db = new ResumeMatchDBEntities()) { var resumeEntity = db.ResumeComplete.FirstOrDefault(f => f.Id == data.Id); if (resumeEntity == null) { LogFactory.Warn($"找不到简历!Id:{data.Id}", MessageSubjectEnum.ZhaoPinGou); return(dataResult); } var userEntity = db.User.FirstOrDefault(f => f.Id == user.Id); if (userEntity == null) { LogFactory.Warn($"找不到用户!用户:{user.Email}", MessageSubjectEnum.ZhaoPinGou); db.TransactionSaveChanges(); return(dataResult); } userEntity.FolderCode = user.FolderCode; if (!unLockResult.IsSuccess) { resumeEntity.Status = 5; if (unLockResult.Code == ResultCodeEnum.NoDownloadNumber) { var userTemp = user; var users = userDictionary.Keys.Where(f => f.Email == userTemp.Email); foreach (var item in users) { userDictionary.TryRemove(item, out cookie); } user.DownloadNumber = 0; userEntity.DownloadNumber = 0; resumeEntity.Status = 2; user.LastLoginTime = DateTime.UtcNow; userEntity.LastLoginTime = DateTime.UtcNow; db.TransactionSaveChanges(); goto Next; } db.TransactionSaveChanges(); return(unLockResult); } resumeEntity.DownloadTime = DateTime.UtcNow; resumeEntity.Status = 4; resumeEntity.UserId = user.Id; user.DownloadNumber--; if (user.DownloadNumber == 0) { userEntity.LastLoginTime = DateTime.UtcNow; } userEntity.DownloadNumber = user.DownloadNumber; dataResult = ResumeDetailSpider(data.MatchResumeId, cookie, host, userToken?.Value); if (!dataResult.IsSuccess) { resumeEntity.Status = 7; db.TransactionSaveChanges(); return(dataResult); } var resumeHtml = dataResult.Data; var cellphone = Regex.IsMatch(resumeHtml, "(?s)电话:</label>(\\d+)</p>") ? Regex.Match(resumeHtml, "(?s)电话:</label>(\\d+)</p>").Result("$1") : null; if (cellphone == null) { resumeEntity.Status = 7; LogFactory.Warn($"补全简历异常!电话为空,ResumeId:{data.ResumeId}", MessageSubjectEnum.ZhaoPinGou); db.TransactionSaveChanges(); return(dataResult); } var email = Regex.IsMatch(resumeHtml, "(?s)邮箱:</label>(.+?)</p>") ? Regex.Match(resumeHtml, "(?s)邮箱:</label>(.+?)</p>").Result("$1") : null; var name = Regex.IsMatch(resumeHtml, "(?s)'resumeb-head-top'><h2>(.+?)</h2><p>") ? Regex.Match(resumeHtml, "(?s)'resumeb-head-top'><h2>(.+?)</h2><p>").Result("$1") : null; resumeEntity.PostBackStatus = 2; resumeEntity.Status = 6; if (resumeEntity.Name.Trim() == name?.Trim()) { var matchedResult = new List <ResumeMatchResult>(); matchedResult.Add(new ResumeMatchResult { ResumeNumber = data.ResumeNumber, Cellphone = cellphone, Email = email, Status = 2 }); if (ApiBase.PostResumes(matchedResult)) { resumeEntity.PostBackStatus = 1; } } else { LogFactory.Warn($"姓名校验异常!库中简历姓名:{resumeEntity.Name},下载简历姓名:{name}"); resumeEntity.Name += $"_{name}"; resumeEntity.PostBackStatus = 0; resumeEntity.Status = 8; } resumeEntity.Email = email; resumeEntity.Cellphone = cellphone; db.TransactionSaveChanges(); dataResult.IsSuccess = true; } return(dataResult); }
/// <summary> /// 登录获取 Cookie /// </summary> /// <param name="email"></param> /// <param name="passWord"></param> /// <param name="host"></param> public static DataResult <CookieContainer> Login(string email, string passWord, string host) { var cookie = new CookieContainer(); var result = new DataResult <CookieContainer>(); var jumpsTimes = 0; Jumps: var param = $"username={email}&password={passWord}&remember=on"; var dataResult = RequestFactory.QueryRequest("http://www.jianlika.com/Index/login.html", param, RequestEnum.POST, cookie, host: host); if (!dataResult.IsSuccess) { result.IsSuccess = false; result.ErrorMsg = $"用户登录异常!异常用户:{email}"; return(result); } var jObject = JsonConvert.DeserializeObject(dataResult.Data) as JObject; if (jObject != null) { if ((int)jObject["status"] != 1) { ++jumpsTimes; if (jumpsTimes < 2) { using (var db = new ResumeMatchDBEntities()) { var user = db.User.FirstOrDefault(f => f.Email == email); if (user != null) { user.IsEnable = false; db.TransactionSaveChanges(); } } result.IsSuccess = false; result.ErrorMsg = $"用户登录异常!异常用户:{email},异常信息:{(string)jObject["message"]}"; return(result); } goto Jumps; } result.Data = cookie; var retryCount = 0; Retry: try { RefreshFreeDownloadNumber(email, cookie, host); } catch (Exception ex) { retryCount++; if (retryCount < 2) { goto Retry; } LogFactory.Error($"刷新简历下载数异常!异常信息:{ex.Message} 堆栈信息:{ex.TargetSite}", MessageSubjectEnum.JianLiKa); } return(result); } return(result); }
/// <summary> /// 登录获取 Cookie /// </summary> /// <param name="email"></param> /// <param name="passWord"></param> /// <param name="host"></param> public static DataResult <CookieContainer> Login(string email, string passWord, string host) { var cookie = new CookieContainer(); var result = new DataResult <CookieContainer>(); var jumpsTimes = 0; Jumps: var param = $"userName={email}&password={passWord}&code=&clientNo=&userToken=&clientType=2"; var dataResult = RequestFactory.QueryRequest("http://qiye.zhaopingou.com/zhaopingou_interface/security_login?timestamp=" + BaseFanctory.GetUnixTimestamp(), param, RequestEnum.POST, cookie, host: host); if (!dataResult.IsSuccess) { result.IsSuccess = false; result.ErrorMsg = $"用户登录异常!异常用户:{email}"; return(result); } var jObject = JsonConvert.DeserializeObject(dataResult.Data) as JObject; if (jObject != null) { if ((int)jObject["errorCode"] != 1) { ++jumpsTimes; if (jumpsTimes > 1) { if (((string)jObject["message"]).Contains("用户不存在")) { using (var db = new ResumeMatchDBEntities()) { var user = db.User.FirstOrDefault(f => f.Email == email); if (user != null) { user.IsEnable = false; db.TransactionSaveChanges(); } } } result.IsSuccess = false; result.ErrorMsg = $"用户登录异常!异常用户:{email},异常信息:{(string)jObject["message"]}"; return(result); } goto Jumps; } cookie.Add(new Cookie { Name = "hrkeepToken", Value = jObject["user"]?["user_token"].ToString(), Domain = ".zhaopingou.com" }); cookie.Add(new Cookie { Name = "zhaopingou_select_city", Value = "-1", Domain = ".zhaopingou.com" }); result.Data = cookie; var retryCount = 0; Retry: try { RefreshFreeDownloadNumber(email, cookie, host); } catch (Exception ex) { retryCount++; if (retryCount < 2) { goto Retry; } LogFactory.Error($"刷新简历下载数异常!异常信息:{ex.Message} 堆栈信息:{ex.TargetSite}", MessageSubjectEnum.ZhaoPinGou); } return(result); } return(result); }
/// <summary> /// 激活 /// </summary> private void Activation() { var users = new List <User>(); lock (lockObj) { using (var db = new ResumeMatchDBEntities()) { db.UsingTransaction(() => { var time = DateTime.UtcNow.AddHours(-1); users = db.User.Where(w => w.Status == 0 && w.Platform == 4 /*&& (!w.IsLocked || w.IsLocked && w.LockedTime < time)*/).ToList(); foreach (var user in users) { user.IsLocked = true; user.LockedTime = DateTime.UtcNow; } db.SaveChanges(); }); } } if (users.Count == 0) { return; } #region 获取未读邮件列表 var seenUids = new List <string>(); var messages = EmailFactory.FetchUnseenMessages("pop.exmail.qq.com", 995, true, Global.Email, Global.PassWord, seenUids); using (var db = new ResumeMatchDBEntities()) { var userIdArr = users.Select(s => s.Id).ToArray(); var userList = db.User.Where(w => userIdArr.Any(a => a == w.Id)); foreach (var user in userList) { user.IsLocked = false; var message = messages.FirstOrDefault(f => f.message.Headers.To.FirstOrDefault()?.Address == user.Email && f.message.Headers.From.Address == "*****@*****.**"); if (message == null) { continue; } var content = Encoding.UTF8.GetString(message.message.FindFirstHtmlVersion().Body); if (!Regex.IsMatch(content, "(?s)完成验证.+?com/track/click/(.+?html)")) { LogFactory.Error($"未匹配到激活码!响应源:{content}", MessageSubjectEnum.ZhaoPinGou); continue; } var url = "http://sctrack.info.zhaopingou.com/track/click/" + Regex.Match(content, "(?s)完成验证.+?com/track/click/(.+?html)\".style").Result("$1"); var host = string.Empty; //if (Global.IsEnanbleProxy) //{ // if (!string.IsNullOrWhiteSpace(user.Host)) // { // host = user.Host; // GetProxy("ZPG_Activation",user.Host); // } //} var dataResult = Activation(url, user.Email, host); //ReleaseProxy("ZPG_Activation", host); if (dataResult == null) { continue; } if (!dataResult.IsSuccess) { LogFactory.Error(dataResult.ErrorMsg, MessageSubjectEnum.ZhaoPinGou); continue; } user.Status = 1; LogFactory.Info($"激活成功!邮箱:{user.Email}", MessageSubjectEnum.ZhaoPinGou); if (!EmailFactory.DeleteMessageByMessageId("pop.exmail.qq.com", 995, true, Global.Email, Global.PassWord, message.message.Headers.MessageId)) { LogFactory.Error($"删除激活邮件失败,邮箱地址:{user.Email}", MessageSubjectEnum.ZhaoPinGou); } } db.TransactionSaveChanges(); } #endregion }
public void MatchZhaopin() { var resumeQueue = new ConcurrentQueue <OldResumeSummary>(); var cookieQueue = new ConcurrentQueue <QueueParam>(); var todayString = DateTime.Today.ToString("yyyy-MM-dd"); using (var db = new MangningXssDBEntities()) { var companyArr = db.ZhaoPinCompany.Where(w => w.Source.Contains("MANUAL")).Select(s => s.Id).ToArray(); var paramArr = db.ZhaopinStaff.Where(w => companyArr.Any(a => a == w.CompanyId) && !string.IsNullOrEmpty(w.Cookie)).Select(s => new { s.CompanyId, s.Cookie }).ToArray(); foreach (var param in paramArr) { var task = db.ZhaopinResumeMatchLimit.FirstOrDefault(f => f.CompanyId == param.CompanyId); if (task == null) { continue; } var todayTask = db.ZhaopinResumeMatchStatistic.FirstOrDefault(f => f.CompanyId == param.CompanyId && f.Date == todayString); if (todayTask == null) { todayTask = new ZhaopinResumeMatchStatistic { Date = todayString, CompanyId = param.CompanyId, MatchedCount = 0, SearchCount = 0, WatchCount = 0 }; db.ZhaopinResumeMatchStatistic.Add(todayTask); db.SaveChanges(); } if (todayTask.SearchCount == task.DailySearchCount || todayTask.WatchCount == task.DailyWatchCount) { continue; } cookieQueue.Enqueue(new QueueParam { CompanyId = param.CompanyId, Cookie = param.Cookie, DailySeachCount = task.DailySearchCount, DailyWatchCount = task.DailyWatchCount, MatchCount = todayTask.MatchedCount, SeachCount = todayTask.SearchCount, WatchCount = todayTask.WatchCount + todayTask.MatchedCount }); } } Task.Run(() => { while (true) { if (resumeQueue.Count < 10) { using (var db = new ResumeMatchDBEntities()) { var resumeList = db.OldResumeSummary.Where(w => w.ResumeId.Length == 10 && !w.IsMatched && w.Status == 0).Take(1000).ToList(); resumeList.ForEach(f => { resumeQueue.Enqueue(f); }); } } Thread.Sleep(TimeSpan.FromSeconds(1)); } }); for (var j = 0; j < 16; j++) { Task.Run(() => { using (var db = new ResumeMatchDBEntities()) { while (true) { #region 匹配 OldResumeSummary resume; if (!resumeQueue.TryDequeue(out resume)) { continue; } var startNum = 0; var resumeTemp = resume; resume = db.OldResumeSummary.FirstOrDefault(f => f.Id == resumeTemp.Id); QueueParam cookie; if (!cookieQueue.TryDequeue(out cookie)) { return; } try { using (var xdb = new MangningXssDBEntities()) { var filePath = $@"E:\Data\智联招聘\{resume.Template}\{resume.ResumeId}.{Path.GetFileNameWithoutExtension(resume.Template)}"; if (!File.Exists(filePath)) { LogFactory.Warn($"指定路径不存在!ResumeNumber=>{resume.ResumeId} Path=>{filePath}"); resume.Status = 1; resume.MatchTime = DateTime.Now; db.SaveChanges(); continue; } var sourceCode = File.ReadAllText(filePath); //var sourceCode = string.Empty; var genderMatch = Regex.Match(sourceCode, "(男|女)"); var gender = string.Empty; if (genderMatch.Success) { gender = genderMatch.Value == "男" ? "1" : "2"; } var matchs = Regex.Matches(sourceCode, "[\u4e00-\u9fa5]{4,11}有限公司[\u4e00-\u9fa5]{0,6}"); var companys = new List <string>(); if (matchs.Count == 0) { if (cookie.SeachCount - cookie.WatchCount < 50) { Console.WriteLine($"该简历未匹配到公司!ResumeNumber=> {resume.ResumeId}"); resume.Status = 1; resume.MatchTime = DateTime.Now; db.SaveChanges(); cookieQueue.Enqueue(cookie); continue; } companys.Add(string.Empty); } for (var i = 0; i < matchs.Count; i++) { if (i == 2) { break; } companys.Add(matchs[i].Value); } var isMatched = true; var age = string.Empty; for (var i = 0; i < companys.Count; i++) { var companyName = companys[i]; var statistic = xdb.ZhaopinResumeMatchStatistic.FirstOrDefault(f => f.Date == todayString && f.CompanyId == cookie.CompanyId); if (statistic == null) { continue; } var cookieContainer = cookie.Cookie.Serialize(".zhaopin.com"); var start = string.IsNullOrEmpty(companyName) ? startNum * 100 : 0; var paramDictionary = new Dictionary <string, string> { { "keywords", "的" }, { "startNum", $"{start}" }, { "rowsCount", "100" }, { "sortColumnName", "sortUpDate" }, { "sortColumn", "sortUpDate desc" }, { "onlyHasImg", "false" }, { "anyKeyWord", "false" }, { "sex", gender }, { "companyName", companyName }, { "onlyLastWork", "false" }, { "ageStart", age }, { "ageEnd", age } }; var requestResult = HttpClientFactory.RequestForString("https://ihr.zhaopin.com/resumesearch/search.do", HttpMethod.Post, paramDictionary, cookieContainer); if (!requestResult.IsSuccess) { isMatched = false; break; } var jObject = JsonConvert.DeserializeObject(requestResult.Data) as JObject; if (jObject["code"] != null) { LogFactory.Warn($"CompanyId => {cookie.CompanyId} 搜索异常 异常信息:{(string)jObject["message"]}"); isMatched = false; return; } statistic.SearchCount += 1; ++cookie.SeachCount; Console.WriteLine($"CompanyId => {cookie.CompanyId} 搜索公司=> {companyName} 性别=> {genderMatch.Value} 今日请求次数:{cookie.SeachCount} "); xdb.SaveChanges(); var matchResumes = (JArray)jObject?["results"]; if (matchResumes == null) { continue; } if (matchResumes.Count == 0) { continue; } var matchResume = matchResumes.FirstOrDefault(a => ((string)a["number"]).Substring(0, 10) == resume.ResumeId); var resumes = matchResumes.Where(w => DateTime.Parse((string)w["modifyDate"]) > DateTime.Today.AddDays(-2)); foreach (var item in resumes) { var wcount = matchResume == null ? cookie.WatchCount : cookie.WatchCount + 1; if (cookie.SeachCount <= wcount) { break; } var number = ((string)item["number"]).Substring(0, 10); if (xdb.ZhaopinResume.Any(a => a.RandomNumber == number)) { continue; } lock (lockObj) { if (xdb.ZhaopinWatchedResume.Any(a => a.ResumeNumber == number)) { continue; } requestResult = HttpClientFactory.RequestForString($"http://ihr.zhaopin.com/resumesearch/getresumedetial.do?resumeNo={(string)item["id"]}_1&resumeSource=1&key=&{(string)item["valResumeTimeStr"]}", HttpMethod.Get, null, cookieContainer); if (requestResult.IsSuccess) { var resumeData = JsonConvert.DeserializeObject <dynamic>(requestResult.Data).data; var resumeDetail = JsonConvert.DeserializeObject(resumeData.detialJSonStr.ToString()); var resumeid = resumeData.resumeId != null ? (int)resumeData.resumeId : resumeDetail.ResumeId != null ? (int)resumeDetail.ResumeId : 0; statistic.WatchCount += 1; Console.WriteLine($"CompanyId => {cookie.CompanyId} 查看简历成功!查看简历份数:{++cookie.WatchCount} ResumeNumber => {number}"); xdb.ZhaopinWatchedResume.Add(new ZhaopinWatchedResume { Id = resumeid, ResumeNumber = number, CompanyId = cookie.CompanyId, WatchTime = DateTime.UtcNow }); xdb.SaveChanges(); } } } if (cookie.SeachCount - cookie.WatchCount > 50 && string.IsNullOrEmpty(companyName) && matchResume == null) { i--; if (startNum == 40) { if (string.IsNullOrEmpty(age)) { age = "18"; } else { age = (Convert.ToInt32(age) + 1).ToString(); } startNum = 0; } else { startNum++; } continue; } if (matchResume == null) { continue; } var resumeNo = (string)matchResume["id"] + "_1"; var resumeNumber = ((string)matchResume["number"]).Substring(0, 10); if (xdb.ZhaopinResume.Any(a => a.RandomNumber == resumeNumber)) { continue; } var valResumeTimeStr = (string)matchResume["valResumeTimeStr"]; requestResult = HttpClientFactory.RequestForString($"http://ihr.zhaopin.com/resumesearch/getresumedetial.do?resumeNo={resumeNo}&resumeSource=1&key=&{valResumeTimeStr}", HttpMethod.Get, null, cookieContainer); if (!requestResult.IsSuccess) { continue; } ++cookie.WatchCount; var resumeJson = JsonConvert.DeserializeObject <dynamic>(requestResult.Data).data; var detail = JsonConvert.DeserializeObject(resumeJson.detialJSonStr.ToString()); var resumeId = resumeJson.resumeId != null ? (int)resumeJson.resumeId : detail.ResumeId != null ? (int)detail.ResumeId : 0; if (xdb.ZhaopinMatchedResume.Any(a => a.Id == resumeId)) { statistic.MatchedCount += 1; LogFactory.Warn($"匹配到重复简历!ResumeId=>{resumeId} ResumeNumber=>{resumeNumber}"); continue; } if (resumeId == 0) { LogFactory.Warn($"CompanyId => {cookie.CompanyId} 解析异常!ResumeId 为空, ResumeNumber:{resumeNumber}"); continue; } var userId = (int)resumeJson.userDetials.userMasterId; var user = xdb.ZhaopinUser.FirstOrDefault(f => f.Id == userId); if (user != null) { if (!user.Source.Contains("MANUAL")) { user.Id = userId; user.Source = "XSS"; user.ModifyTime = BaseFanctory.GetTime((string)detail.DateModified).ToUniversalTime(); user.CreateTime = BaseFanctory.GetTime((string)detail.DateCreated).ToUniversalTime(); user.Cellphone = resume.Cellphone; user.Email = resume.Email; user.Name = resumeJson.userDetials.userName.ToString(); user.UpdateTime = DateTime.UtcNow; user.Username = resumeJson.userDetials.email.ToString(); } } else { xdb.ZhaopinUser.Add(new ZhaopinUser { Id = userId, Source = "XSS", ModifyTime = BaseFanctory.GetTime((string)detail.DateModified).ToUniversalTime(), CreateTime = BaseFanctory.GetTime((string)detail.DateCreated).ToUniversalTime(), Cellphone = resume.Cellphone, Email = resume.Email, Name = resumeJson.userDetials.userName.ToString(), UpdateTime = DateTime.UtcNow }); } var resumeEntity = xdb.ZhaopinResume.FirstOrDefault(f => f.Id == resumeId); if (resumeEntity == null) { xdb.ZhaopinResume.Add(new ZhaopinResume { Id = resumeId, RandomNumber = resumeNumber, UserId = userId, RefreshTime = BaseFanctory.GetTime((string)detail.DateModified).ToUniversalTime(), UpdateTime = DateTime.UtcNow, UserExtId = detail.UserMasterExtId.ToString(), DeliveryNumber = resumeEntity?.DeliveryNumber, Source = resumeEntity == null ? "XSS" : resumeEntity.Source }); } else { resumeEntity.RandomNumber = resumeNumber; resumeEntity.UserId = userId; resumeEntity.RefreshTime = BaseFanctory.GetTime((string)detail.DateModified).ToUniversalTime(); resumeEntity.UpdateTime = DateTime.UtcNow; resumeEntity.UserExtId = detail.UserMasterExtId.ToString(); resumeEntity.DeliveryNumber = resumeEntity?.DeliveryNumber; resumeEntity.Source = resumeEntity == null ? "XSS" : resumeEntity.Source; } var path = $"{ConfigurationManager.AppSettings["Resume.SavePath"]}{DateTime.Now:yyyyMMdd}"; if (!Directory.Exists(path)) { Directory.CreateDirectory(path); } var resumePath = $@"{path}\{resumeId}.json"; resumeJson.userDetials.email = resume.Email; resumeJson.userDetials.mobilePhone = resume.Cellphone; File.WriteAllText(resumePath, JsonConvert.SerializeObject(resumeJson)); uploadOssActionBlock.Post(resumePath); statistic.MatchedCount += 1; Console.WriteLine($"CompanyId => {cookie.CompanyId} 搜索简历成功!匹配成功 {++cookie.MatchCount} 份, ResumeNumner=>{resumeNumber},ResumeId=>{resumeId}"); xdb.ZhaopinMatchedResume.Add(new ZhaopinMatchedResume { Id = resumeId, CompanyId = cookie.CompanyId, MatchTime = DateTime.UtcNow }); xdb.SaveChanges(); resume.IsMatched = true; break; } if (cookie.SeachCount < cookie.DailySeachCount) { cookieQueue.Enqueue(cookie); } if (isMatched) { resume.Status = 99; resume.MatchTime = DateTime.Now; } db.SaveChanges(); } } catch (Exception ex) { //while (true) //{ // if(ex.InnerException == null) break; // ex = ex.InnerException; //} if (cookie.SeachCount < cookie.DailySeachCount) { cookieQueue.Enqueue(cookie); } LogFactory.Warn($"程序异常, 异常信息:{ex.Message} 堆栈:{ex.StackTrace}"); } #endregion } } }); } }
/// <summary> /// 注册 /// </summary> private void Register() { var proxyId = 0; var host = string.Empty; var isAddProxy = false; lock (lockObj) { using (var db = new ResumeMatchDBEntities()) { var data = db.UsingTransaction(() => { var result = new DataResult(); if (!Global.IsEnanbleProxy) { return(result); } var ipArr = db.Proxy.AsNoTracking() .GroupBy(g => g.Host) .Select(s => new { Host = s.Key, Count = s.Count() }) .Where(w => w.Count < Global.PlatformCount * Global.PlatformHostCount) .Select(s => s.Host).ToArray(); if (ipArr.Length > 0) { var proxy = db.Proxy.FirstOrDefault(f => ipArr.Any(a => a == f.Host) && f.Platform == 4 && !string.IsNullOrEmpty(f.Host) && f.Count < Global.PlatformHostCount); if (proxy != null) { proxyId = proxy.Id; host = proxy.Host; proxy.Count++; var user = db.User.Where(w => w.Host == host && w.Platform == 4).OrderBy(o => o.DownloadNumber).FirstOrDefault(); if (user == null) { LogFactory.Error($"找不到用户,Host = {host}", MessageSubjectEnum.ZhaoPinGou); result.IsSuccess = false; return(result); } db.SaveChanges(); return(result); } } var proxyEntity = new Proxy { Count = 1, Platform = 4 }; db.Proxy.Add(proxyEntity); db.SaveChanges(); isAddProxy = true; proxyId = proxyEntity.Id; return(result); }); if (!data.IsSuccess) { return; } } } if (Global.IsEnanbleProxy) { if (string.IsNullOrWhiteSpace(host)) { host = GetProxy("ZPG_Register", true); } else { GetProxy("ZPG_Register", host); } } var dataResult = Register(host); ReleaseProxy("ZPG_Register", host); using (var db = new ResumeMatchDBEntities()) { var proxy = db.Proxy.FirstOrDefault(f => f.Id == proxyId); if (dataResult == null || !dataResult.IsSuccess) { if (proxy != null) { if (isAddProxy) { db.Proxy.Remove(proxy); } else { proxy.Count--; } } } else { db.User.Add(new User { Email = dataResult.Data.Email, Password = dataResult.Data.Password, InviteCode = dataResult.Data.InviteCode, CreateTime = DateTime.UtcNow, IsEnable = true, DownloadNumber = 10, Host = host, Platform = 4, Status = 0, IsLocked = false, LockedTime = new DateTime(1900, 1, 1), RequestNumber = 0 }); if (proxy != null) { proxy.Host = host; } } db.TransactionSaveChanges(); } }
private static List <ResumeSearch> PullAllResumes() { var list = PullResumesByZeLin(); if (list != null) { return(list); } var resumesList = new List <ResumeSearch>(); Retry: var dataResult = RequestFactory.QueryRequest(Global.HostZhao + "/splider/Resume/GetResumeWithNoDeal?rowcount=100"); if (!dataResult.IsSuccess || string.IsNullOrWhiteSpace(dataResult.Data)) { goto Retry; } var jObject = JsonConvert.DeserializeObject(dataResult.Data) as JObject; if (jObject != null) { var jArray = jObject["Resumes"] as JArray; if (jArray == null) { goto Retry; } Global.TotalMatch += jArray.Count; foreach (var item in jArray) { var resume = new ResumeSearch(); resume.Name = item["Name"].ToString(); var worksArr = (JArray)item["Works"]; resume.LastCompany = worksArr.Count > 0 ? worksArr[0]["Company"].ToString() : ""; var educationsArr = (JArray)item["Educations"]; resume.University = educationsArr.Count > 0 ? educationsArr[0]["School"].ToString() : ""; resume.ResumeId = item["Reference"]?["Id"].ToString(); resume.ResumeNumber = ((JArray)item["Reference"]?["Mapping"])?[2]["Value"].ToString(); resume.UserMasterExtId = ((JArray)item["Reference"]?["Mapping"])?[1]["Value"].ToString(); resume.University = item["Educations"]?[0]?["School"].ToString(); resume.Gender = item["Gender"].ToString() == "男" ? (short)0 : item["Gender"].ToString() == "女" ? (short)1 : (short)-1; resume.Degree = item["Degree"].ToString(); resume.Introduction = item["Intention"]?["Evaluation"].ToString(); resumesList.Add(resume); } } if (resumesList.Count == 0) { goto Retry; } using (var db = new ResumeMatchDBEntities()) { var resumeIdArr = resumesList.Select(s => s.ResumeId).ToArray(); var resumes = db.ResumeComplete.Where(w => resumeIdArr.Any(a => a == w.ResumeId)).ToList(); var removeArr = resumes.Select(s => s.ResumeId).ToArray(); if (removeArr.Any()) { LogFactory.Warn("获取到重复简历!ResumeID:" + string.Join(",", removeArr)); } var idArr = resumes.Where(a => a.Status == 6 || a.Status == 2).Select(s => s.ResumeId).ToArray(); if (idArr.Length > 0) { LogFactory.Warn("过滤掉已有联系方式的重复简历!" + string.Join(",", idArr)); resumes.RemoveAll(r => idArr.Any(a => a == r.ResumeId)); resumesList.RemoveAll(r => idArr.Any(a => a == r.ResumeId)); } db.ResumeComplete.RemoveRange(resumes); db.TransactionSaveChanges(); db.ResumeComplete.AddRange(resumesList.Select(s => new ResumeComplete { CreateTime = DateTime.UtcNow, Gender = s.Gender, Introduction = s.Introduction, LastCompany = s.LastCompany, LibraryExist = 0, Name = s.Name, PostBackStatus = 0, ResumeId = s.ResumeId, ResumeNumber = s.ResumeNumber, ResumePlatform = 1, Status = 0, University = s.University, UserMasterExtId = s.UserMasterExtId, Degree = s.Degree, Weights = 0 })); db.TransactionSaveChanges(); } return(FilterExist(resumesList)); }
/// <summary> /// 激活 /// </summary> private static void Activation() { var users = new List <User>(); lock (lockObj) { using (var db = new ResumeMatchDBEntities()) { db.UsingTransaction(() => { var time = DateTime.UtcNow.AddHours(-1); users = db.User.Where(w => w.Status == 0 && w.Platform == 3 && (!w.IsLocked || w.IsLocked && w.LockedTime < time)).ToList(); foreach (var user in users) { user.IsLocked = true; user.LockedTime = DateTime.UtcNow; } db.SaveChanges(); }); } } if (users.Count == 0) { return; } #region 获取未读邮件列表 var seenUids = new List <string>(); var messages = EmailFactory.FetchUnseenMessages("pop.exmail.qq.com", 995, true, Global.Email, Global.PassWord, seenUids); using (var db = new ResumeMatchDBEntities()) { var userIdArr = users.Select(s => s.Id).ToArray(); var userList = db.User.Where(w => userIdArr.Any(a => a == w.Id)); foreach (var user in userList) { user.IsLocked = false; var message = messages.FirstOrDefault(f => f.message.Headers.To.FirstOrDefault()?.Address == user.Email); if (message == null) { continue; } var content = Encoding.Default.GetString(message.message.RawMessage); if (!Regex.IsMatch(content, "(?s)token=(.+?)</a>")) { LogFactory.Error($"未匹配到激活码!响应源:{content}", MessageSubjectEnum.JianLiKa); continue; } var activationCode = Regex.Match(content, "(?s)token=(.+?)</a>").Result("$1").Substring(2); var host = string.Empty; //if (Global.IsEnanbleProxy) //{ // if (!string.IsNullOrWhiteSpace(user.Host)) // { // host = user.Host; // GetProxy("JLK_Activation",user.Host); // } //} var dataResult = Activation(activationCode, user.Email, host); //ReleaseProxy("JLK_Activation", host); if (dataResult == null) { continue; } if (!dataResult.IsSuccess) { LogFactory.Error(dataResult.ErrorMsg, MessageSubjectEnum.JianLiKa); continue; } user.Status = 1; LogFactory.Info($"激活成功!邮箱:{user.Email}", MessageSubjectEnum.JianLiKa); if (!EmailFactory.DeleteMessageByMessageId("pop.exmail.qq.com", 995, true, Global.Email, Global.PassWord, message.message.Headers.MessageId)) { LogFactory.Error($"删除激活邮件失败,邮箱地址:{user.Email}", MessageSubjectEnum.JianLiKa); } } db.TransactionSaveChanges(); } #endregion }
/// <summary> /// 获取用户 /// </summary> /// <param name="userDictionary"></param> /// <param name="host"></param> /// <param name="isMatch"></param> /// <param name="matchPlatform"></param> /// <param name="loginFuc"></param> /// <param name="cookie"></param> /// <returns></returns> public static DataResult <User> GetUser(ConcurrentDictionary <User, CookieContainer> userDictionary, string host, bool isMatch, MatchPlatform matchPlatform, Func <string, string, string, DataResult <CookieContainer> > loginFuc, out CookieContainer cookie) { var dataResult = new DataResult <User>(); var messageSubjectEnum = EnumFactory <MessageSubjectEnum> .Parse(matchPlatform); User user; cookie = null; var dateTime = DateTime.UtcNow.Date.AddHours(-8); using (var db = new ResumeMatchDBEntities()) { if (userDictionary.Keys.All(a => a.Host != host)) { List <User> users; if (isMatch) { users = db.User.Where(w => w.IsEnable && w.Platform == (short)matchPlatform && w.Status == 1 && w.Host == host).ToList(); } else { users = db.User.Where(w => w.IsEnable && w.Platform == (short)matchPlatform && w.Status == 1 /* && w.Host == host*/ && (w.LastLoginTime == null || w.DownloadNumber > 0 || w.LastLoginTime < dateTime)).ToList(); } if (!users.Any()) { dataResult.IsSuccess = false; dataResult.Code = ResultCodeEnum.NoUsers; return(dataResult); } foreach (var item in users) { for (var i = 0; i < 5; i++) { if (userDictionary.TryAdd(item, null)) { break; } if (i == 4) { LogFactory.Warn($"向字典中添加用户 {item.Email} 失败!", messageSubjectEnum); } } } } Next: if (isMatch) { var userQuery = userDictionary.Keys.Where(f => f.IsEnable && f.Host == host); if (!string.IsNullOrWhiteSpace(host)) { userQuery = userQuery.Where(w => w.RequestDate == null || w.RequestDate.Value.Date < DateTime.UtcNow.Date || w.RequestDate.Value.Date == DateTime.UtcNow.Date && w.RequestNumber < Global.TodayMaxRequestNumber); } user = userQuery.OrderBy(o => o.RequestNumber).FirstOrDefault(); } else { user = userDictionary.Keys .Where(f => f.IsEnable /*&& f.Host == host */ && (f.LastLoginTime == null || f.DownloadNumber > 0 || f.LastLoginTime < dateTime)) .OrderBy(o => o.Email) .FirstOrDefault(); } if (user == null) { dataResult.IsSuccess = false; if (isMatch) { dataResult.Code = ResultCodeEnum.RequestUpperLimit; } else { dataResult.Code = ResultCodeEnum.NoUsers; } LogFactory.Warn(JsonConvert.SerializeObject(userDictionary), messageSubjectEnum); var list = userDictionary.Keys.Where(w => w.Host == host); foreach (var item in list) { for (var i = 0; i < 5; i++) { if (userDictionary.TryRemove(item, out cookie)) { break; } if (i == 4) { LogFactory.Warn($"从字典中移除用户 {item.Email} 失败!", messageSubjectEnum); dataResult.ErrorMsg += $"向字典中移除用户 {item.Email} 失败!"; return(dataResult); } } } return(dataResult); } if (isMatch) { if (user.RequestDate == null || user.RequestDate.Value.Date < DateTime.UtcNow.Date) { user.RequestDate = DateTime.UtcNow.Date; user.RequestNumber = 0; } user.RequestNumber++; } for (var i = 0; i < 5; i++) { if (userDictionary.TryGetValue(user, out cookie)) { break; } } if (cookie == null) { var result = loginFuc(user.Email, user.Password, host); if (!result.IsSuccess) { LogFactory.Warn(result.ErrorMsg, messageSubjectEnum); userDictionary.TryRemove(user, out cookie); dataResult.IsSuccess = false; return(dataResult); } cookie = result.Data; if (cookie != null) { for (var i = 0; i < 5; i++) { if (userDictionary.TryUpdate(user, cookie, null)) { break; } } } } if (cookie == null) { goto Next; } var userEntity = db.User.FirstOrDefault(f => f.Id == user.Id); if (userEntity != null) { userEntity.RequestDate = user.RequestDate; userEntity.RequestNumber = user.RequestNumber; } db.TransactionSaveChanges(); } dataResult.Data = user; return(dataResult); }
public List <SearchResumeModel> GetOldResumes() { var searchResumeList = new List <SearchResumeModel>(); using (var db = new ResumeMatchDBEntities()) { var resumeList = db.OldResumeSummary.Where(w => w.ResumeId.Length == 11 && !w.IsMatched && (w.Status == 0 || w.Status == 99)).Take(500).ToList(); foreach (var resume in resumeList) { resume.Status = 2; } db.SaveChanges(); foreach (var resume in resumeList) { var searchResume = new SearchResumeModel(); var filePath = $@"\\DOLPHIN-PC\Data\智联招聘\{resume.Template}\{resume.ResumeId}.{Path.GetFileNameWithoutExtension(resume.Template)}"; if (!File.Exists(filePath)) { LogFactory.Warn($"指定路径不存在!ResumeNumber=>{resume.ResumeId} Path=>{filePath}"); resume.Status = 1; resume.MatchTime = DateTime.Now; db.SaveChanges(); continue; } var sourceCode = File.ReadAllText(filePath); var genderMatch = Regex.Match(sourceCode, "(男|女)"); if (genderMatch.Success) { searchResume.Gender = genderMatch.Value; } var replaceMatch = Regex.Match(sourceCode, "应聘机构:.+?>(.+?)</strong>"); if (replaceMatch.Success) { sourceCode = sourceCode.Replace(replaceMatch.Result("$1"), ""); } var matchs = Regex.Matches(sourceCode, "([\u4e00-\u9fa5]{4,11}有限公司[\u4e00-\u9fa5]{0,6})"); if (matchs.Count == 0) { Console.WriteLine($"该简历未匹配到公司!ResumeNumber=> {resume.ResumeId}"); resume.Status = 1; resume.MatchTime = DateTime.Now; db.SaveChanges(); continue; } var companys = new List <string>(); for (var i = 0; i < matchs.Count; i++) { if (i == 2) { break; } companys.Add(matchs[i].Result("$1")); } searchResume.Companys = companys; searchResume.Cellphone = resume.Cellphone; searchResume.Email = resume.Email; searchResume.SearchResumeId = resume.ResumeId; searchResumeList.Add(searchResume); } } return(searchResumeList); }
public static DataResult <string> GetResumeId(ResumeSearch data, string host) { var dataResult = new DataResult <string>(); CookieContainer cookie; var result = GetUser(userDictionary, host, true, MatchPlatform.JianLiKa, Login, out cookie); if (!result.IsSuccess) { dataResult.IsSuccess = false; dataResult.Code = result.Code; dataResult.ErrorMsg = result.ErrorMsg; return(dataResult); } var user = result.Data; var count = 0; var keyWord = string.Empty; if (!string.IsNullOrWhiteSpace(data.University)) { keyWord += data.University + " "; } ++count; if (count == 0) { return(new DataResult <string>()); } if (!string.IsNullOrWhiteSpace(data.Introduction)) { var strArr = jbs.Cut(data.Introduction.Replace("\r\n", "")).Where(w => w.Length > 1).ToArray(); keyWord += string.Join(" ", strArr.Skip(strArr.Length / 2 - 1).Take(3 - count)); } keyWord = HttpUtility.UrlEncode(keyWord)?.Replace("+", "%20"); var gender = data.Gender; var degress = MatchDegree(data.Degree); dataResult = GetResumeId(keyWord, data.LastCompany, gender, degress, data.Introduction, cookie, user); using (var db = new ResumeMatchDBEntities()) { var resume = db.ResumeComplete.FirstOrDefault(f => f.ResumeId == data.ResumeId); if (resume != null) { resume.Host = host; resume.MatchPlatform = (short)MatchPlatform.JianLiKa; resume.MatchTime = DateTime.UtcNow; resume.UserId = user.Id; if (!string.IsNullOrWhiteSpace(dataResult.Data)) { resume.Status = 2; resume.MatchResumeId = dataResult.Data; } db.TransactionSaveChanges(); } } return(dataResult); }