private void ExportCD()
{
var cellphones = File.ReadAllLines(@"D:\给过的手机号-副本.txt");
using (var db = new MangningXssDBEntities())
{
var beginDataTime = DateTime.Today.AddDays(-4);
var endDataTime = DateTime.Today.AddDays(1);
var dataList = db.DodiBusiness
.Join(db.DodiUserInfomation, a => a.Id, b => b.BusinessId, (B, A) => new { B.Id, A.UserName, A.GraduatedSchool, B.BranchOffice, A.Email, A.Cellphone, B.CreateTime, B.Sources, A.JobName, B.PromoteBrand, A.ProfessionalTitle })
.Where(w => /*!w.BranchOffice.StartsWith("北京") && !w.BranchOffice.StartsWith("上海") && !w.BranchOffice.StartsWith("广州") && */ w.BranchOffice.StartsWith("成都") && w.CreateTime > beginDataTime && w.CreateTime < endDataTime)
.Select(s => new { s.Id, s.UserName, s.Cellphone, s.Email, s.Sources, s.CreateTime, s.JobName, s.ProfessionalTitle })
.ToList();
const string cookie = "PHPSESSID=j0lklef94l9akqabg41n3nqd93; Example_auth=f9d7XYivszgUGkEXygbytRrg8EzZWngyS25FZaKx1OSub%2FhBVliH; Hm_lvt_1360b6fe7fa346ff51189adc58afb874=1507336367,1507510768,1507596684,1507682510; Hm_lpvt_1360b6fe7fa346ff51189adc58afb874=1507705480";
var cookieContainer = cookie.Serialize("crm.dodi.cn");
HttpClientFactory.RequestForString("http://crm.dodi.cn/index.php/Notice/index", HttpMethod.Get, null, cookieContainer);
var sb = new StringBuilder();
sb.AppendLine("姓名\t手机\t邮箱\t年龄\t学历\t更新日期\t性别\t平台\t地点\t职位\t专业");
var index = 0;
foreach (var item in dataList)
{
var response = HttpClientFactory.RequestForString($"http://crm.dodi.cn/index.php/Main/khxxy/business_id/{item.Id}/source/false_note", HttpMethod.Get, null, cookieContainer);
if (!response.IsSuccess)
{
RunInMainthread(() =>
{
Program.SetLog(this.tbx_Log, $"ID:{item.Id} 请求失败！{response.ErrorMsg}");
});
LogFactory.Warn($"ID:{item.Id} 请求失败！{response.ErrorMsg}");
continue;
}
if (!response.Data.Contains("商 机 ID："))
{
RunInMainthread(() =>
{
Program.SetLog(this.tbx_Log, $"ID:{item.Id} 商机为空！{response.ErrorMsg}");
});
LogFactory.Warn($"ID:{item.Id} 商机为空！{response.ErrorMsg}");
continue;
}
var match = Regex.Match(response.Data, @"resume_email\('(.*?)','(\d+)','(\d+)','(\d+)',(\d+)\)");
if (!match.Success)
{
RunInMainthread(() =>
{
Program.SetLog(this.tbx_Log, $"ID:{item.Id} 匹配详情失败！");
});
LogFactory.Warn($"ID:{item.Id} 匹配详情失败！");
continue;
}
var email = HttpUtility.UrlEncode(match.Result("$1"));
var phone = HttpUtility.UrlEncode(match.Result("$2"));
var email_id = HttpUtility.UrlEncode(match.Result("$3"));
var now_month = HttpUtility.UrlEncode(match.Result("$4"));
var school_id = HttpUtility.UrlEncode(match.Result("$5"));
if (cellphones.Contains(phone))
{
continue;
}
response = HttpClientFactory.RequestForString($"http://crm.dodi.cn/index.php/Main/email_body?email={email}&phone={phone}&email_id={email_id}&now_month={now_month}&school_id={school_id}", HttpMethod.Get, null, cookieContainer);
if (!response.IsSuccess)
{
RunInMainthread(() =>
{
Program.SetLog(this.tbx_Log, $"ID:{item.Id} 详情请求失败！{response.ErrorMsg}");
});
LogFactory.Warn($"ID:{item.Id} 详情请求失败！{response.ErrorMsg}");
continue;
}
//File.WriteAllText($@"D:\Business\{item.Id}.txt", response.Data);
var data = Regex.Unescape(response.Data);
var xbMatch = Regex.Match(data, "(男|女)");
if (xbMatch.Value.Trim() == "女")
{
continue;
}
var ageMatch = Regex.Match(data, "((?<=[^0-9])[0-9]{2}岁|年龄\\s*[0-9]{2}(?=[^0-9]))");
if (!ageMatch.Success)
{
ageMatch = Regex.Match(data, "(19|20)[0-9]{2}(年|-)[0-9]{1,2}(月|)(?=[^0-9])");
}
//var xlMatch = Regex.Match(data, "(高中|初中|小学|大专|中专)");
var xlMatch = Regex.Match(data, "(大专|本科|硕士|博士|MBA)");
//var date = item.CreateTime < DateTime.Today.AddDays(-3) ? item.CreateTime?.AddDays(2) : item.CreateTime;
sb.AppendLine($"{item.UserName}\t{item.Cellphone}\t{item.Email}\t{ageMatch.Value}\t{xlMatch.Value}\t{item.CreateTime?.ToString("yyyy-MM-dd")}\t{xbMatch.Value}\t{item.Sources.Substring(item.Sources.LastIndexOf("_", StringComparison.Ordinal) + 1)}\t成都\t{item.JobName}\t{item.ProfessionalTitle}");
//sb.AppendLine($"{item.Cellphone}");
//if (index % 1000 == 0)
//{
// File.WriteAllText(@"D:\非北上广.txt", sb.ToString());
//}
RunInMainthread(() =>
{
Program.SetLog(this.tbx_Log, $"{Interlocked.Increment(ref index)}/{dataList.Count}");
});
}
;
File.WriteAllText(@"D:\成都大专以上.txt", sb.ToString());
}
}
private void OldSystemLoginForm_Load(object sender, EventArgs e)
{
FiddlerApplication.BeforeRequest += oSessions =>
{
//if (oSessions.url.Contains("zhaopin")) oSessions["X-OverrideGateway"] = $"{webProxyIp}:{webProxyPort}";
oSessions.bBufferResponse = true;
if (oSessions.url.Contains("captcha.js"))
{
if (oSessions.RequestHeaders.Exists("If-Modified-Since"))
{
oSessions.RequestHeaders.Remove("If-Modified-Since");
}
if (oSessions.RequestHeaders.Exists("If-None-Match"))
{
oSessions.RequestHeaders.Remove("If-None-Match");
}
if (oSessions.RequestHeaders.Exists("Accept-Encoding"))
{
oSessions.RequestHeaders.Remove("Accept-Encoding");
}
}
if (oSessions.RequestHeaders.Exists("User-Agent"))
{
oSessions.RequestHeaders["User-Agent"] = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.84 Safari/537.36";
}
};
FiddlerApplication.BeforeResponse += oSessions =>
{
if (oSessions.url.Contains("captcha.js"))
{
oSessions.ResponseHeaders["Cache-Control"] = "no-store";
var jsContent = oSessions.GetResponseBodyEncoding().GetString(oSessions.ResponseBody);
jsContent = "var hackStr = '';" + jsContent;
jsContent = jsContent.Replace("pData.join(\";\")", "hackStr");
jsContent = jsContent.Remove(jsContent.Length - 4);
jsContent += "this.hack = function (coordinate){ $(\"#captcha-submitCode\").removeClass(\"btn-disabled\");validate = true;hackStr = coordinate; $(\"#captcha-submitCode\").trigger(\"click\"); return true;}";
jsContent += "}\r\n";
jsContent += "function execHack(coordinate){this.captcha.hack(coordinate); return 1;}";
oSessions.ResponseBody = oSessions.GetResponseBodyEncoding().GetBytes(jsContent);
}
};
if (!FiddlerApplication.IsStarted())
{
FiddlerApplication.Startup(8887, FiddlerCoreStartupFlags.Default);
}
List <ZhaopinCleaningProcedure> accounts;
using (var db = new MangningXssDBEntities())
{
accounts = db.ZhaopinCleaningProcedure.Where(w => w.IsEnable && (string.IsNullOrEmpty(w.Cookie) || w.StartTime < DateTime.Today)).ToList();
}
this.AsyncSetLog(this.tbx_Log, $"共 {accounts.Count} 个号准备登录！");
this.webBrowser.DocumentCompleted += webBrowser_DocumentCompleted;
this.webBrowser.ScriptErrorsSuppressed = true;
this.RunAsync(() =>
{
foreach (var item in accounts)
{
account = item.Account;
password = item.Password;
webBrowser.Navigate("https://passport.zhaopin.com/org/login");
while (true)
{
if (!isWaitLogin)
{
Thread.Sleep(1000);
continue;
}
var index = 0;
object obj;
var isChecked = false;
while (!isChecked)
{
this.Invoke((MethodInvoker) delegate
{
index++;
if (this.webBrowser.Document?.GetElementById("CheckCodeCapt")?.GetAttribute("value") == "验证通过")
{
Thread.Sleep(3000);
this.webBrowser.Document?.GetElementById("loginbutton")?.InvokeMember("click");
isChecked = true;
return;
}
if (index % 2 != 0)
{
obj = this.webBrowser.Document?.InvokeScript("execHack", new object[] { "66,76;173,44;239,80" });
}
else
{
obj = this.webBrowser.Document?.InvokeScript("execHack", new object[] { "44,44;190,37;122,48" });
}
if (obj == null)
{
this.AsyncSetLog(this.tbx_Log, "调用验证 JS 异常");
}
});
Thread.Sleep(500);
}
while (true)
{
if (!isLogined)
{
Thread.Sleep(1000);
continue;
}
break;
}
this.Invoke((MethodInvoker) delegate
{
item.Cookie = webBrowser.Document?.Cookie;
var document = webBrowser.Document;
document?.ExecCommand("ClearAuthenticationCache", false, null);
});
this.AsyncSetLog(this.tbx_Log, $"{account} 获取Cookie成功！");
using (var db = new MangningXssDBEntities())
{
var pro = db.ZhaopinCleaningProcedure.FirstOrDefault(f => f.Id == item.Id);
if (pro != null)
{
pro.Cookie = item.Cookie;
}
db.SaveChanges();
}
isWaitLogin = false;
isLogined = false;
break;
}
}
});
}
public void Watch()
{
//using (var db = new MangningXssDBEntities())
//{
// var maxId = db.ZhaopinCleaningProcedure.Max(m => m.Id);
// for (var i = 1; i <= 11; i++)
// {
// db.ZhaopinCleaningProcedure.Add(new ZhaopinCleaningProcedure
// {
// Id = (short)(i + maxId),
// Account = $"qhzl_{i:000}",
// Cookie = null,
// IsEnable = true,
// IsOnline = false,
// Password = "******",
// StartTime = DateTime.Now.AddDays(-1)
// });
// }
// db.SaveChanges();
//}
if (!Directory.Exists(savePath))
{
Directory.CreateDirectory(savePath);
}
var cookieQueue = new ConcurrentQueue <KeyValuePair <KeyValuePair <int, string>, CookieContainer> >();
var companyDic = new ConcurrentDictionary <string, int>();
using (var db = new MangningXssDBEntities())
{
var companyArr = db.ZhaoPinCompany.Where(w => w.Source.Contains("MANUAL")).Select(s => s.Id).ToArray();
var paramArr = db.ZhaopinStaff.Where(w => companyArr.Any(a => a == w.CompanyId) && !string.IsNullOrEmpty(w.Cookie)).Select(s => new { s.CompanyId, s.Username, s.Cookie }).ToArray();
foreach (var item in paramArr)
{
companyDic[item.Username] = 0;
cookieQueue.Enqueue(new KeyValuePair <KeyValuePair <int, string>, CookieContainer>(new KeyValuePair <int, string>(item.CompanyId, item.Username), item.Cookie.Serialize(".zhaopin.com")));
}
}
using (var db = new MangningXssDBEntities())
{
var list = db.ZhaopinCleaningProcedure.Where(w => w.IsEnable && !w.IsOnline && !string.IsNullOrEmpty(w.Cookie));
foreach (var item in list)
{
if (item.StartTime < DateTime.Today)
{
item.TodayWatch = 0;
}
companyDic[item.Account] = item.TodayWatch;
cookieQueue.Enqueue(new KeyValuePair <KeyValuePair <int, string>, CookieContainer>(new KeyValuePair <int, string>(0, item.Account), item.Cookie.Serialize(".zhaopin.com")));
item.IsOnline = true;
item.StartTime = DateTime.Now;
}
db.SaveChanges();
}
Console.WriteLine($"已获取 Cookie 数 => {cookieQueue.Count}");
for (var j = 0; j < 16; j++)
{
Task.Run(() =>
{
try
{
while (true)
{
KeyValuePair <KeyValuePair <int, string>, CookieContainer> temp;
if (!cookieQueue.TryDequeue(out temp))
{
Thread.Sleep(1000);
continue;
}
while (true)
{
var condition = GetSingleCondition(temp.Key.Value);
if (condition == null)
{
Console.WriteLine($"Company => {temp.Key.Value} 找不到可搜索的条件！");
Thread.Sleep(1000);
continue;
}
var pageIndex = 1;
var pageTotal = 1;
var total = 0;
var isbreak = false;
var workYearsEnd = condition.WorkYears == 30 ? 30 : condition.WorkYears + 1;
while (pageIndex < pageTotal + 1)
{
var url = $"https://ihrsearch.zhaopin.com/Home/ResultForCustom?SF_1_1_8={condition.Age}%2C{condition.Age}&SF_1_1_9={condition.Gender}&SF_1_1_4={condition.WorkYears}%2C{workYearsEnd}&SF_1_1_5={condition.Degrees}%2C{condition.Degrees}&orderBy=BIRTH_YEAR%2C0&SF_1_1_27=0&SF_1_1_7={updateRange}%2C9&exclude=1&pageIndex={pageIndex}&pageSize=60";
var requestResult = RequestFactory.QueryRequest(url, cookieContainer: temp.Value, referer: url);
if (!requestResult.IsSuccess)
{
LogFactory.Warn($"Company => {temp.Key.Value} 条件搜索异常！异常原因=>{requestResult.ErrorMsg} Condition=>{JsonConvert.SerializeObject(condition)}");
continue;
}
if (pageTotal == 1)
{
var match = Regex.Match(requestResult.Data, "(?s)<span>(\\d+)</span>份简历.+?rd-resumelist-pageNum\">1/(\\d+)</span>");
if (!match.Success)
{
if (requestResult.Data.Contains("text/javascript\" r='m'"))
{
LogFactory.Warn($"Cookie 过期，过期用户 => {temp.Key.Value}");
RemoveCookie(temp.Key.Value, companyDic[temp.Key.Value]);
}
else
{
LogFactory.Warn($"Company => {temp.Key.Value} 条件搜索异常！返回页面解析异常！{requestResult.Data}");
}
isbreak = true;
break;
}
total = Convert.ToInt32(match.Result("$1"));
pageTotal = Convert.ToInt32(match.Result("$2"));
if (total == 0)
{
Console.WriteLine("该条件无搜索结果！");
break;
}
}
Console.WriteLine($"Company => {temp.Key.Value} 第 {pageIndex} 页 共 {pageTotal} 页 {total} 个结果");
pageIndex++;
//var matchs = Regex.Matches(requestResult.Data, "RedirectToRd/(.+?)','(.+?)','(.+?)',this\\);this");
var matchs = Regex.Matches(requestResult.Data, "(?s)RedirectToRd/([^\r\n]+?)','([^\r\n]+?)','([^\r\n]+?)',this\\);this.+?(\\d{2}-\\d{2}-\\d{2})");
var index = 0;
foreach (Match item in matchs)
{
var number = item.Result("$1").Substring(0, 10);
var numberParam = item.Result("$1").Substring(0, item.Result("$1").IndexOf("?", StringComparison.Ordinal));
DateTime updateDateTime;
if (DateTime.TryParse(item.Result("$4"), out updateDateTime))
{
if (QueryResumeIsExists(number, updateDateTime))
{
continue;
}
}
else
{
if (QueryResumeIsExists(number))
{
continue;
}
}
requestResult = RequestFactory.QueryRequest($"http://ihr.zhaopin.com/resumesearch/getresumedetial.do?resumeNo={numberParam}&searchresume=1&resumeSource=1&keyword=%E8%AF%B7%E8%BE%93%E5%85%A5%E7%AE%80%E5%8E%86%E5%85%B3%E9%94%AE%E8%AF%8D%EF%BC%8C%E5%A4%9A%E5%85%B3%E9%94%AE%E8%AF%8D%E5%8F%AF%E7%94%A8%E7%A9%BA%E6%A0%BC%E5%88%86%E9%9A%94&t={item.Result("$2")}&k={item.Result("$3")}&v=undefined&version=3&openFrom=1", cookieContainer: temp.Value);
if (!requestResult.IsSuccess)
{
LogFactory.Warn($"Company => {temp.Key.Value} 简历详情查看异常！异常原因=>{requestResult.ErrorMsg} ResumeNumber=>{number}");
continue;
}
try
{
var jsonObj = JsonConvert.DeserializeObject <dynamic>(requestResult.Data);
if ((int)jsonObj["code"] != 1)
{
LogFactory.Warn($"Company => {temp.Key.Value} ResumeNumber => {number} 查看详情异常 信息：{(string)jsonObj["message"]} 查看简历份数：{companyDic[temp.Key.Value]}");
if (((string)jsonObj["message"]).Contains("当日查看简历已达上限"))
{
RemoveCookie(temp.Key.Value, companyDic[temp.Key.Value]);
isbreak = true;
break;
}
continue;
}
var resumeData = jsonObj.data;
SaveCacheByOss(new KeyValuePair <KeyValuePair <int, string>, dynamic>(temp.Key, resumeData));
Console.WriteLine($"Company => {temp.Key.Value} 查看简历成功！查看简历份数：{++companyDic[temp.Key.Value]} ResumeNumber => {number} {++index}/{matchs.Count}");
}
catch (Exception ex)
{
while (true)
{
if (ex.InnerException == null)
{
break;
}
ex = ex.InnerException;
}
LogFactory.Error($"保存简历信息异常！异常信息：{ex.Message} 响应信息：{requestResult.Data}");
}
//if (companyDic[temp.Key.Value] > 1500)
//{
// RemoveCookie(temp.Key.Value, companyDic[temp.Key.Value]);
// isbreak = true;
// break;
//}
}
if (isbreak)
{
break;
}
}
if (isbreak)
{
break;
}
SetSearchStatus(condition.Id, 1, total);
}
}
}
catch (Exception ex)
{
while (true)
{
if (ex.InnerException == null)
{
break;
}
ex = ex.InnerException;
}
LogFactory.Error($"Watch异常！异常信息：{ex.Message} {ex.StackTrace}");
}
});
}
}
/// <summary>
/// 上传简历
/// </summary>
private static void UploadResume()
{
var serializer = new Serialization.Template.Zhaopin.Json.v1.Serializer();
var stopwatch = new Stopwatch();
while (true)
{
string path;
if (!uploadQueue.TryDequeue(out path))
{
Thread.Sleep(100);
continue;
}
Interlocked.Increment(ref totalUpload);
try
{
stopwatch.Restart();
var resumeId = Convert.ToInt32(Path.GetFileNameWithoutExtension(path));
var content = File.ReadAllText(path);
var jsonObj = JsonConvert.DeserializeObject <dynamic>(content);
dynamic formatterResume;
try
{
if (jsonObj.Flag != null && jsonObj.Flag < 0)
{
formatterResume = Format.Convert_V0(Format.ConvertTo_Dtl_V5(content));
}
else
{
var serializationResume = serializer.Deserialize(content);
formatterResume = Formatter.Template.Zhaopin.Json.v1.Formatter.Format(serializationResume);
}
}
catch (Exception ex)
{
var filePath = $"{ConfigurationManager.AppSettings["File.FailPath"]}{Path.GetFileName(path)}";
if (File.Exists(filePath))
{
File.Delete(filePath);
}
File.Move(path, filePath);
Trace.TraceError(ex.ToString());
using (var db = new MangningXssDBEntities())
{
var resume = db.ZhaopinResume.FirstOrDefault(f => f.Id == resumeId);
if (resume != null)
{
resume.Flag = 0xA;
}
db.SaveChanges();
}
continue;
}
string tag;
while (true)
{
var data = JsonConvert.SerializeObject(new
{
formatterResume.Reference.Id,
formatterResume.Reference.Source
});
var response = PrepareUploadResume(data);
if (response.Code.ToString() != "0")
{
FinishUploadResume(data);
Trace.WriteLine($"{DateTime.Now} > PrepareUploadResume failed ! Data = {data}, Response = {JsonConvert.SerializeObject(response)}");
continue;
}
var badoucaiResumeJson = JsonConvert.SerializeObject(formatterResume);
response = UploadResume(badoucaiResumeJson);
var returnCode = (string)response.Code;
using (var db = new MangningXssDBEntities())
{
tag = response.Reference?.Tag.ToString();
db.ZhaopinResumeUploadLog.Add(new ZhaopinResumeUploadLog
{
ResumeId = resumeId,
ReturnCode = returnCode,
Tag = tag,
UploadTime = DateTime.Now
});
FinishUploadResume(data);
var resume = db.ZhaopinResume.FirstOrDefault(f => f.Id == resumeId);
if (resume != null)
{
resume.Flag = 0xF;
}
if (jsonObj.Flag != null && jsonObj.Flag < 0)
{
resume.Flag = 0xB;
}
if (returnCode != "0")
{
Trace.WriteLine($"{DateTime.Now} > UploadResume failed !ResumeId = {resumeId}, Response = {JsonConvert.SerializeObject(response)}.");
tag = "NULL";
db.SaveChanges();
var filePath = $"{ConfigurationManager.AppSettings["File.UploadFailPath"]}{Path.GetFileName(path)}";
if (File.Exists(filePath))
{
File.Delete(filePath);
}
File.Move(path, filePath);
break;
}
if (tag == "C" || tag == "U" || tag == "N")
{
using (var stream = new MemoryStream(GZip.Compress(Encoding.UTF8.GetBytes(badoucaiResumeJson))))
{
var id = response.Reference.ResumeId.ToString();
badoucaiOssClient.PutObject(badoucaiBucketName, $"Badoucai/{id}", stream);
}
ProgramTasksThread.HandleResume(content, resumeId); // 更新 Resume 表的信息
}
File.Delete(path);
db.SaveChanges();
Interlocked.Increment(ref successUpload);
}
break;
}
stopwatch.Stop();
Console.WriteLine($"{DateTime.Now} > ResumeId = {resumeId}, Tag = {tag}, {successUpload}/{totalUpload} = {Math.Round(successUpload / (double)totalUpload, 3) * 100}%, Elapsed = {stopwatch.ElapsedMilliseconds} ms.");
}
catch (Exception ex)
{
Trace.TraceError(ex.ToString());
}
}
}
/// <summary>
/// 导入外部Josn
/// </summary>
private static void ImportResume()
{
var importCount = 0;
while (true)
{
try
{
var filePaths = Directory.GetFiles(importFilePath);
if (filePaths.Length == 0)
{
Thread.Sleep(10 * 1000);
continue;
}
Console.WriteLine($"{DateTime.Now} > {filePaths.Length} Need Import ! ");
foreach (var filePath in filePaths)
{
try
{
var jsonObj = JsonConvert.DeserializeObject <dynamic>(File.ReadAllText(filePath));
var resumeData = jsonObj.data == null ? jsonObj : jsonObj.data;
var resumeDetail = JsonConvert.DeserializeObject(resumeData.detialJSonStr.ToString());
var resumeId = resumeData.resumeId != null ? (int)resumeData.resumeId : resumeDetail.ResumeId != null ? (int)resumeDetail.ResumeId : 0;
var cellphone = resumeData.userDetials.mobilePhone.ToString();
if (string.IsNullOrEmpty(cellphone))
{
var path = importFailPath + Path.GetFileName(filePath);
if (File.Exists(path))
{
File.Delete(path);
}
File.Move(filePath, path);
continue;
}
var resumeNumber = ((string)resumeData.resumeNo).Substring(0, 10);
var userId = (int)resumeData.userDetials.userMasterId;
var refreshTime = BaseFanctory.GetTime((string)resumeDetail.DateLastReleased).ToUniversalTime();
using (var db = new MangningXssDBEntities())
{
var resume = db.ZhaopinResume.FirstOrDefault(f => f.Id == resumeId);
var isNeedUpload = true;
var sourceFlag = -1;
var isUpload = false;
if (resume != null)
{
sourceFlag = resume.Flag;
}
if (!(resume?.RefreshTime != null && resume.RefreshTime.Value.Date >= refreshTime.Date) || resume?.Flag < 8)
{
if (resume != null)
{
resume.RandomNumber = resumeNumber;
resume.RefreshTime = refreshTime;
resume.UpdateTime = DateTime.UtcNow;
if (string.IsNullOrEmpty(resume.UserExtId))
{
resume.UserExtId = resumeDetail.UserMasterExtId?.ToString();
}
if (resumeData.Flag != null && (int)resumeData.Flag < 0)
{
if (resume.Flag >= 8 || mangningOssClient.DoesObjectExist(mangningBucketName, $"Zhaopin/Resume/{resume.Id}") || mangningOssClient.DoesObjectExist(mangningBucketName, $"WatchResume/{resume.Id}"))
{
isNeedUpload = false;
}
}
resume.Flag = 0xE;
}
else
{
resume = new ZhaopinResume
{
Id = resumeId,
RandomNumber = resumeNumber,
UserId = userId,
RefreshTime = refreshTime,
UpdateTime = DateTime.UtcNow,
UserExtId = resumeDetail.UserMasterExtId.ToString(),
DeliveryNumber = null,
Source = "Import",
Flag = 0xE
};
db.ZhaopinResume.Add(resume);
}
var user = db.ZhaopinUser.FirstOrDefault(f => f.Id == userId);
if (user != null)
{
if (!user.Source.Contains("MANUAL"))
{
user.ModifyTime = BaseFanctory.GetTime((string)resumeDetail.DateModified).ToUniversalTime();
user.CreateTime = resumeData.Flag != null && (int)resumeData.Flag < 0 ? user.CreateTime : BaseFanctory.GetTime((string)resumeDetail.DateCreated).ToUniversalTime();
user.Cellphone = resumeData.userDetials.mobilePhone.ToString();
user.Email = resumeData.userDetials.email.ToString();
user.Name = resumeData.userDetials.userName.ToString();
user.UpdateTime = DateTime.UtcNow;
user.Username = resumeData.userDetials.email.ToString();
}
}
else
{
user = new ZhaopinUser
{
Id = userId,
Source = "Import",
ModifyTime = BaseFanctory.GetTime((string)resumeDetail.DateModified).ToUniversalTime(),
CreateTime = BaseFanctory.GetTime((string)resumeDetail.DateCreated).ToUniversalTime(),
Cellphone = resumeData.userDetials.mobilePhone.ToString(),
Email = resumeData.userDetials.email.ToString(),
Name = resumeData.userDetials.userName.ToString(),
UpdateTime = DateTime.UtcNow,
Username = resumeData.userDetials.email.ToString()
};
db.ZhaopinUser.Add(user);
}
if (isNeedUpload)
{
isUpload = true;
var resumeContent = JsonConvert.SerializeObject(resumeData);
using (var jsonStream = new MemoryStream(GZip.Compress(Encoding.UTF8.GetBytes(resumeContent))))
{
mangningOssClient.PutObject(mangningBucketName, $"Zhaopin/Resume/{resume.Id}", jsonStream);
}
}
db.SaveChanges();
}
File.Delete(filePath);
Console.WriteLine($"{DateTime.Now} > Improt success ! ResumeId = {resumeId}, SourceFlag = {sourceFlag}, IsUpload = {isUpload}, Count = {++importCount}.");
}
}
catch (Exception ex)
{
var path = importFailPath + Path.GetFileName(filePath);
if (File.Exists(path))
{
File.Delete(path);
}
File.Move(filePath, path);
Trace.WriteLine($"{DateTime.Now} > Import Error Message = {ex.Message}, Path = {path}.");
}
}
}
catch (Exception ex)
{
Trace.TraceError(ex.ToString());
}
}
}
/// <summary>
/// 处理多迪简历
/// </summary>
private static void HandleDodiResume()
{
while (true)
{
ZhaopinCleaningProcedure cleaningProcedure;
if (!cookieQueue.TryDequeue(out cleaningProcedure))
{
Thread.Sleep(100);
continue;
}
var localCookieContainer = cleaningProcedure.Cookie.Serialize(".zhaopin.com");
while (true)
{
int businessId;
if (!businessIdQueue.TryDequeue(out businessId))
{
Thread.Sleep(100);
continue;
}
using (var db = new MangningXssDBEntities())
{
var dodiBusiness = db.DodiBusiness.FirstOrDefault(f => f.Id == businessId);
if (dodiBusiness != null)
{
dodiBusiness.Status = 2;
db.SaveChanges();
}
}
try
{
var response = RequestFactory.QueryRequest($"http://crm.dodi.cn/index.php/Main/khxxy/business_id/{businessId}/source/false_note", cookieContainer: cookieContainer);
if (!response.IsSuccess)
{
continue;
}
if (!response.Data.Contains("商 机 ID："))
{
Trace.WriteLine($"{DateTime.Now} > Business ID is Empty ! BusinessId = {businessId}");
continue;
}
var match = Regex.Match(response.Data, @"resume_email\('(.*?)','(\d+)','(\d+)','(\d+)',(\d+)\)");
if (!match.Success)
{
Trace.WriteLine($"{DateTime.Now} > Details match failed ! BusinessId = {businessId}");
continue;
}
var email = HttpUtility.UrlEncode(match.Result("$1"));
var phone = HttpUtility.UrlEncode(match.Result("$2"));
var email_id = HttpUtility.UrlEncode(match.Result("$3"));
var now_month = HttpUtility.UrlEncode(match.Result("$4"));
var school_id = HttpUtility.UrlEncode(match.Result("$5"));
response = HttpClientFactory.RequestForString($"http://crm.dodi.cn/index.php/Main/email_body?email={email}&phone={phone}&email_id={email_id}&now_month={now_month}&school_id={school_id}", HttpMethod.Get, null, cookieContainer);
if (!response.IsSuccess)
{
Trace.WriteLine($"{DateTime.Now} > Details request failed ! BusinessId = {businessId}");
continue;
}
var resumeDetail = Regex.Unescape(response.Data);
var company = Regex.Match(resumeDetail, "20[01]\\d[.][\\d\\s-.]+(.*?)\\s*（").ResultOrDefault("$1", "");
if (string.IsNullOrEmpty(company))
{
Console.WriteLine($"{DateTime.Now} > Company is Empty ! BusinessId = {businessId}");
continue;
}
var matchs = Regex.Matches(resumeDetail, "20[01]\\d\\.\\d{2}\\s-\\s20[01]\\d\\.\\d{2}\\s+([^&]+?)&");
var project = string.Empty;
foreach (Match item in matchs)
{
var matchString = item.Result("$1");
if (matchString.Contains("（"))
{
continue;
}
if (matchString.Contains("</br>"))
{
continue;
}
if (matchString.Contains(" "))
{
continue;
}
project = matchString;
break;
}
if (string.IsNullOrEmpty(project))
{
Console.WriteLine($"{DateTime.Now} > Project is Empty ! BusinessId = {businessId}");
continue;
}
var url = $"https://ihrsearch.zhaopin.com/Home/ResultForCustom?SF_1_1_1={HttpUtility.UrlEncode(project)}&SF_1_1_25=COMPANY_NAME_ALL:{HttpUtility.UrlEncode(company)}&SF_1_1_27=0&orderBy=DATE_MODIFIED,1&pageSize=60&exclude=1";
var requestResult = RequestFactory.QueryRequest(url, cookieContainer: localCookieContainer, referer: url);
if (!requestResult.IsSuccess)
{
Trace.TraceError($"{DateTime.Now} > Search request failed ! BusinessId = {businessId}.");
continue;
}
match = Regex.Match(requestResult.Data, "(?s)<span>(\\d+)</span>份简历.+?rd-resumelist-pageNum\">1/(\\d+)</span>");
if (!match.Success)
{
if (requestResult.Data.Contains("text/javascript\" r='m'"))
{
Console.WriteLine($"{DateTime.Now} > Cookie Expired ! Account = {cleaningProcedure.Account}.");
}
else
{
Trace.TraceWarning($"{DateTime.Now} > Condition search error ! Page content = {requestResult.Data}, Account = {cleaningProcedure.Account}.");
}
using (var db = new MangningXssDBEntities())
{
var procedure = db.ZhaopinCleaningProcedure.FirstOrDefault(f => f.Id == cleaningProcedure.Id);
if (procedure != null)
{
procedure.Cookie = null;
db.SaveChanges();
}
}
businessIdQueue.Enqueue(businessId);
break;
}
var total = Convert.ToInt32(match.Result("$1"));
if (total != 1)
{
Console.WriteLine($"{DateTime.Now} > Search match wrong ! ResultCount = {total}, BusinessId = {businessId}, Company = {company}, Project = {project}.");
continue;
}
match = Regex.Match(requestResult.Data, "(?s)RedirectToRd/([^\r\n]+?)','([^\r\n]+?)','([^\r\n]+?)',this\\);this.+?(\\d{2}-\\d{2}-\\d{2})");
HandleResume(match, HttpUtility.HtmlDecode(phone), HttpUtility.HtmlDecode(email), cleaningProcedure, businessId);
}
catch (Exception ex)
{
Trace.TraceError(ex.ToString());
}
}
}
}
/// <summary>
/// 查看简历详情
/// </summary>
/// <param name="match"></param>
/// <param name="localCookieContainer"></param>
/// <param name="cellphone"></param>
/// <param name="email"></param>
/// <param name="businessId"></param>
/// <returns></returns>
private static bool WatchResumeDetail(Match match, CookieContainer localCookieContainer, string cellphone, string email, int businessId)
{
var number = match.Result("$1").Substring(0, 10);
var numberParam = match.Result("$1").Substring(0, match.Result("$1").IndexOf("?", StringComparison.Ordinal));
var requestResult = RequestFactory.QueryRequest($"http://ihr.zhaopin.com/resumesearch/getresumedetial.do?resumeNo={numberParam}&searchresume=1&resumeSource=1&keyword=%E8%AF%B7%E8%BE%93%E5%85%A5%E7%AE%80%E5%8E%86%E5%85%B3%E9%94%AE%E8%AF%8D%EF%BC%8C%E5%A4%9A%E5%85%B3%E9%94%AE%E8%AF%8D%E5%8F%AF%E7%94%A8%E7%A9%BA%E6%A0%BC%E5%88%86%E9%9A%94&t={match.Result("$2")}&k={match.Result("$3")}&v=undefined&version=3&openFrom=1", cookieContainer: localCookieContainer);
if (!requestResult.IsSuccess)
{
Trace.WriteLine($"{DateTime.Now} > Watching error ! Message = {requestResult.ErrorMsg}, ResumeNumber = {number}");
return(false);
}
try
{
var jsonObj = JsonConvert.DeserializeObject <dynamic>(requestResult.Data);
if ((int)jsonObj["code"] != 1)
{
Trace.WriteLine($"{DateTime.Now} > Watching failure ! Message = {(string)jsonObj["message"]}, ResumeNumber = {number}");
return(false);
}
var resumeData = jsonObj.data;
resumeData.userDetials.mobilePhone = cellphone;
resumeData.userDetials.email = email;
var resumeDetail = JsonConvert.DeserializeObject(resumeData.detialJSonStr.ToString());
var resumeId = resumeData.resumeId != null ? (int)resumeData.resumeId : resumeDetail.ResumeId != null ? (int)resumeDetail.ResumeId : 0;
using (var db = new MangningXssDBEntities())
{
var business = db.DodiBusiness.FirstOrDefault(f => f.Id == businessId);
if (business != null)
{
business.Status = 3;
db.SaveChanges();
}
}
File.WriteAllText($"{path}{resumeId}.json", JsonConvert.SerializeObject(resumeData));
Console.WriteLine($"{DateTime.Now} > Completion success ! ResumeNumber = {number}.");
return(true);
}
catch (Exception ex)
{
while (true)
{
if (ex.InnerException == null)
{
break;
}
ex = ex.InnerException;
}
Trace.WriteLine($"{DateTime.Now} > Watching exception ! Message = {ex.Message}, ResumeNumber = {number}");
return(false);
}
}
/// <summary>
/// 刷新验证码
/// </summary>
/// <returns></returns>
private DataResult ReferenceCheckCode()
{
// 准备颜色变换矩阵的元素
float[][] colorMatrixElements =
{
new float[] { -1, 0, 0, 0, 0 },
new float[] { 0, -1, 0, 0, 0 },
new float[] { 0, 0, -1, 0, 0 },
new float[] { 0, 0, 0, 1, 0 },
new float[] { 1, 1, 1, 0, 1 }
};
// 为 ImageAttributes 设置颜色变换矩阵
var colorMatrix = new ColorMatrix(colorMatrixElements);
var imageAttributes = new ImageAttributes();
imageAttributes.SetColorMatrix(colorMatrix, ColorMatrixFlag.Default, ColorAdjustType.Bitmap);
locationStack.Clear();
var streamResult = HttpClientFactory.RequestForStream("https://rd.zhaopin.com/resumePreview/captcha/getcap?t1515065150298", HttpMethod.Get, cookieContainer: cookieContainer);
if (!streamResult.IsSuccess)
{
this.RunInMainthread(() =>
{
this.lbl_Tip.Text = $"提示：{DateTime.Now:HH:mm:ss} 未获取到验证码！{streamResult.ErrorMsg}";
});
isGetCheckCode = false;
return(new DataResult("未获取到验证码!"));
}
var codeStream = streamResult.Data;
timestamp = (DateTime.Now.ToUniversalTime().Ticks - 621355968000000000) / 10000000 + new Random().Next(0, 1000).ToString("000");
// 将 ImageAttributes 应用于绘制
Bitmap picBody;
try
{
picBody = Image.FromHbitmap(ImageHelper.GetValidCode_Zhaopin(Image.FromStream(codeStream)).GetHbitmap());
}
catch (Exception)
{
using (var db = new MangningXssDBEntities())
{
var checkCode = db.ZhaopinCheckCode.FirstOrDefault(f => f.HandleUser == handleUser && f.Status != 2);
if (checkCode != null)
{
db.ZhaopinCheckCode.Remove(checkCode);
db.SaveChanges();
}
this.RunInMainthread(() =>
{
this.lbl_Tip.Text = $"提示：{DateTime.Now:HH:mm:ss} Cookie 过期！CompanyName = {compnayName}";
});
isGetCheckCode = false;
return(new DataResult("未获取到验证码!"));
}
}
var imageBody = new Bitmap(picBody.Width, picBody.Height);
var graphics = Graphics.FromImage(imageBody);
graphics.DrawImage(picBody, new Rectangle(0, 0, picBody.Width, picBody.Height), 0, 0, picBody.Width, picBody.Height, GraphicsUnit.Pixel, imageAttributes);
var picHeader = Image.FromHbitmap(ImageHelper.GetValidCodeSource_Zhaopin(Image.FromStream(codeStream)).GetHbitmap());
var imageHeader = new Bitmap(picHeader.Width, picHeader.Height);
graphics = Graphics.FromImage(imageHeader);
graphics.DrawImage(picHeader, new Rectangle(0, 0, picHeader.Width, picHeader.Height), 0, 0, picHeader.Width, picHeader.Height, GraphicsUnit.Pixel, imageAttributes);
graphics.Dispose();
this.pic_Header.Image = imageHeader;
this.pic_Body.Image = imageBody;
endTime = DateTime.Now.AddMinutes(1);
locationStack.Push(new Watermark
{
Index = 0,
Image = this.pic_Body.Image,
LocationX = -999,
LocationY = -999
});
isGetCheckCode = false;
return(new DataResult());
}
private void btn_Checking_Click(object sender, EventArgs e)
{
var coordinatParam = string.Empty;
var list = new List <Watermark>();
if (!locationStack.Any())
{
return;
}
while (true)
{
var watermark = locationStack.Pop();
if (watermark.Index == 0)
{
break;
}
list.Add(watermark);
}
list.Reverse();
coordinatParam = list.Aggregate(coordinatParam, (current, item) => current + $";{item.LocationX + range / 2},{item.LocationY + range / 2}");
if (coordinatParam.Length == 0)
{
coordinatParam = "99,99;99,99;99,99";
}
coordinatParam = coordinatParam.Substring(1);
var checkedResult = RequestFactory.QueryRequest("https://rd.zhaopin.com/resumePreview/captcha/verifyjsonp?callback=jsonpCallback", $"p={HttpUtility.UrlEncode(coordinatParam)}&time={timestamp}", RequestEnum.POST, cookieContainer, host: proxyHost);
if (!checkedResult.IsSuccess)
{
MessageBox.Show(checkedResult.ErrorMsg);
return;
}
var match = Regex.Match(checkedResult.Data, "jsonpCallback\\('(.+?)'\\)");
if (!match.Success)
{
this.lbl_Tip.Text = $"提示：{DateTime.Now:HH:mm:ss} 验证失败！";
endTime = DateTime.Now.AddMinutes(1);
var dataResult = ReferenceCheckCode();
if (!dataResult.IsSuccess)
{
endTime = null;
}
return;
}
checkedResult = RequestFactory.QueryRequest("https://rd.zhaopin.com/resumePreview/resume/_CheackValidatingCode?validatingCode=" + match.Result("$1"), "", RequestEnum.POST, cookieContainer, host: proxyHost);
if (!checkedResult.IsSuccess || checkedResult.Data.ToLower().Trim() != "true")
{
this.lbl_Tip.Text = $"提示：{DateTime.Now:HH:mm:ss} 验证失败！{checkedResult.ErrorMsg}";
endTime = DateTime.Now.AddMinutes(1);
var dataResult = ReferenceCheckCode();
if (!dataResult.IsSuccess)
{
endTime = null;
}
return;
}
this.lbl_Tip.Text = $"提示：{DateTime.Now:HH:mm:ss} 验证成功！";
endTime = null;
this.btn_GetCheckCode.Enabled = true;
this.btn_ReferenceCheckCode.Enabled = false;
this.btn_Checking.Enabled = false;
this.pic_Body.Image = null;
this.pic_Header.Image = null;
using (var db = new MangningXssDBEntities())
{
var checkCode = db.ZhaopinCheckCode.FirstOrDefault(f => f.HandleUser == handleUser && f.Status == 1);
if (checkCode == null)
{
return;
}
checkCode.Status = 2;
checkCode.CompleteTime = DateTime.Now;
db.SaveChanges();
}
ReferenceRank();
}
/// <summary>
/// 處理簡歷列表
/// </summary>
/// <param name="resumes"></param>
/// <param name="unhandled"></param>
/// <param name="staff"></param>
/// <param name="isWhile"></param>
/// <param name="cookieContainer"></param>
private static void HandleResumes(dynamic resumes, int unhandled, ZhaopinStaff staff, ref bool isWhile, CookieContainer cookieContainer)
{
var stopwatch = new Stopwatch();
foreach (var item in resumes)
{
#region Handle resume
try
{
stopwatch.Restart();
#region Save resume and Upload
var requestResult = RequestFactory.QueryRequest($"http://ihr.zhaopin.com/resumesearch/getresumedetial.do?resumeNo={item.id}_{item.jobNumber}_{item.number}_1_1&resumeSource=3", cookieContainer: cookieContainer);
if (!requestResult.IsSuccess)
{
Trace.TraceWarning(requestResult.ErrorMsg);
continue;
}
var content = JsonConvert.DeserializeObject <dynamic>(requestResult.Data);
if ((int)content.code == 6001) // 登录过期
{
using (var db = new MangningXssDBEntities())
{
var zhaopinStaff = db.ZhaopinStaff.FirstOrDefault(f => f.Id == staff.Id);
if (zhaopinStaff != null)
{
zhaopinStaff.Cookie = null;
db.SaveChanges();
dictionary.TryRemove(staff.Id, out companyId);
}
}
Trace.WriteLine($"{DateTime.Now} > Loging Timeout ! Username = {staff.Username}, Message = {content.message}.");
isWhile = false;
break;
}
if ((int)content.code != 1)
{
Trace.WriteLine($"{DateTime.Now} > Get Resume Detail Error ! Username = {staff.Username}, Message = {content.message}.");
continue;
}
var resumeData = content.data;
var resumeDetail = JsonConvert.DeserializeObject(resumeData.detialJSonStr.ToString());
var refreshTime = BaseFanctory.GetTime((string)resumeDetail.DateLastReleased).ToUniversalTime();
resumeData.detialJSonStr = resumeDetail;
var resumeNumber = ((string)resumeData.resumeNo).Substring(0, 10);
var userId = (int)resumeData.userDetials.userMasterId;
var resumeId = resumeData.resumeId != null ? (int)resumeData.resumeId : resumeDetail.ResumeId != null ? (int)resumeDetail.ResumeId : 0;
var status = "Handle";
using (var db = new MangningXssDBEntities())
{
var resume = db.ZhaopinResume.FirstOrDefault(f => f.Id == resumeId);
if (!(resume?.RefreshTime != null && resume.RefreshTime.Value.Date >= refreshTime.Date))
{
if (resume != null)
{
resume.RandomNumber = resumeNumber;
resume.RefreshTime = refreshTime;
resume.UpdateTime = DateTime.UtcNow;
if (string.IsNullOrEmpty(resume.UserExtId))
{
resume.UserExtId = resumeDetail.UserMasterExtId.ToString();
}
resume.Source = !resume.Source.Contains("Deliver") ? resume.Source += ",Deliver" : resume.Source;
resume.Flag = 0xE;
}
else
{
resume = new ZhaopinResume
{
Id = resumeId,
RandomNumber = resumeNumber,
UserId = userId,
RefreshTime = refreshTime,
UpdateTime = DateTime.UtcNow,
UserExtId = resumeDetail.UserMasterExtId.ToString(),
DeliveryNumber = null,
Source = "Deliver",
Flag = 0xE
};
db.ZhaopinResume.Add(resume);
}
var user = db.ZhaopinUser.FirstOrDefault(f => f.Id == userId);
if (user != null)
{
if (!user.Source.Contains("MANUAL"))
{
user.ModifyTime = BaseFanctory.GetTime((string)resumeDetail.DateModified).ToUniversalTime();
user.CreateTime = BaseFanctory.GetTime((string)resumeDetail.DateCreated).ToUniversalTime();
user.Cellphone = resumeData.userDetials.mobilePhone.ToString();
user.Email = resumeData.userDetials.email.ToString();
user.Name = resumeData.userDetials.userName.ToString();
user.UpdateTime = DateTime.UtcNow;
user.Username = resumeData.userDetials.email.ToString();
}
}
else
{
user = new ZhaopinUser
{
Id = userId,
Source = "Deliver",
ModifyTime = BaseFanctory.GetTime((string)resumeDetail.DateModified).ToUniversalTime(),
CreateTime = BaseFanctory.GetTime((string)resumeDetail.DateCreated).ToUniversalTime(),
Cellphone = resumeData.userDetials.mobilePhone.ToString(),
Email = resumeData.userDetials.email.ToString(),
Name = resumeData.userDetials.userName.ToString(),
UpdateTime = DateTime.UtcNow,
Username = resumeData.userDetials.email.ToString()
};
db.ZhaopinUser.Add(user);
}
var resumeContent = JsonConvert.SerializeObject(resumeData);
using (var jsonStream = new MemoryStream(GZip.Compress(Encoding.UTF8.GetBytes(resumeContent))))
{
mangningOssClient.PutObject(mangningBucketName, $"Zhaopin/Resume/{resumeId}", jsonStream);
}
var resumePath = $"{uploadFilePath}{resumeId}.json";
File.WriteAllText(resumePath, JsonConvert.SerializeObject(resumeData));
db.SaveChanges();
}
else
{
status = "NoHandle";
}
}
#endregion
Thread.Sleep(3000);
#region SignResume
var data = HttpUtility.UrlEncode(JsonConvert.SerializeObject(new
{
signTag = "noSuit",
resumeList = new List <dynamic>
{
new
{
resumeNo = $"{item.jobNumber}_{item.number}_{item.version}_1",
resumenumber = item.number,
item.version,
lanType = 1,
jobname = ((string)resumeData?.jobName)?.Replace("\"", "\\\"").Replace("\\", "\\\\"),
jobno = item.jobNumber,
resumesource = item.resumeSource,
resumeJobId = item.id,
resumerName = ((string)item?.userName).Replace("\r", "").Replace("\n", "").Replace("\"", "\\\""),
resumejlName = ((string)item?.name).Replace("\r", "").Replace("\n", "").Replace("\"", "\\'").Replace("\\", "\\\\"),
resumerId = item.userId
}
},
mark = ""
}));
requestResult = RequestFactory.QueryRequest("https://ihr.zhaopin.com/resumemanage/resumesignstate.do", $"data={data}", RequestEnum.POST, cookieContainer);
if (!requestResult.IsSuccess)
{
Trace.TraceWarning(requestResult.ErrorMsg);
continue;
}
content = JsonConvert.DeserializeObject <dynamic>(requestResult.Data);
if ((int)content.code == 6001) // 登录过期
{
using (var db = new MangningXssDBEntities())
{
var zhaopinStaff = db.ZhaopinStaff.FirstOrDefault(f => f.Id == staff.Id);
if (zhaopinStaff != null)
{
zhaopinStaff.Cookie = null;
db.SaveChanges();
dictionary.TryRemove(staff.Id, out companyId);
}
}
Trace.WriteLine($"{DateTime.Now} > Loging Timeout ! Username = {staff.Username}, Message = {content.message}.");
isWhile = false;
break;
}
if ((int)content.code != 1)
{
Trace.WriteLine($"{DateTime.Now} > Sign Resume Error ! Username = {staff.Username}, Message = {content.message}, ResumeId = {resumeId}.");
continue;
}
#endregion
stopwatch.Stop();
var elapsed = stopwatch.ElapsedMilliseconds;
Interlocked.Increment(ref count);
Console.WriteLine($"{DateTime.Now} > {count} {status} {staff.Username}：Unhandled = {--unhandled}, RId = {resumeId}, Elapsed = {elapsed} ms.");
}
catch (Exception ex)
{
Trace.TraceError(ex.ToString());
}
#endregion
}
}
private static void DownloadForTalentFolder()
{
var business = new SpiderResumeBusiness();
Console.WriteLine("开始获取 Cookie ...");
var cookies = business.GetCookies();
foreach (var item in cookies)
{
cookieQueue.Enqueue(item);
}
for (var i = 0; i < taskCount; i++)
{
tasks.Add(Task.Run(() =>
{
while (true)
{
KeyValuePair <int, string> cookie;
if (!cookieQueue.TryDequeue(out cookie))
{
break;
}
try
{
var token = cookie.Value.Substring(cookie.Value.IndexOf("at=", StringComparison.Ordinal) + 3, 32);
Console.WriteLine($"Cookie 队列剩余：{cookieQueue.Count}, 当前 Token：{token}");
var cookieContainer = cookie.Value.Serialize("zhaopin.com");
DataResult <string> requestResult;
dynamic content;
dynamic summary;
int count;
int pageNumber;
int begin;
int end;
var isBreak = false;
#region 载简历库简历
foreach (var key in new[] { "deal", "commu", "interview", "noSuit" })
{
var isContinue = false;
requestResult = business.GetResumes(cookieContainer, 0, 1, key, token);
if (!requestResult.IsSuccess)
{
break;
}
content = JsonConvert.DeserializeObject <dynamic>(requestResult.Data);
if ((int)content.code != 1)
{
LogFactory.Warn($"简历列表下载异常！message：{content.message} companyId：{cookie.Key}");
if ((int)content.code == 6001) // 登录过期
{
isBreak = true;
using (var db = new MangningXssDBEntities())
{
var staff = db.ZhaopinStaff.FirstOrDefault(f => f.CompanyId == cookie.Key);
if (staff != null)
{
staff.Cookie = null;
db.SaveChanges();
}
}
break;
}
continue;
}
count = (int)content.data[key].numFound;
//Console.WriteLine($"CompanyId：{cookie.Key}，分类：{key} 简历数：{count}");
if (count == 0)
{
continue;
}
if (business.IsDownloadByDeliveryId((long)content.data[key].results[0].id, key))
{
continue;
}
pageNumber = (int)Math.Ceiling(count / 90.0);
while (true)
{
requestResult = business.GetResumes(cookieContainer, count - 1, 90, key, token);
if (!requestResult.IsSuccess)
{
isContinue = true;
break;
}
content = JsonConvert.DeserializeObject <dynamic>(requestResult.Data);
if ((int)content.data[key].results.Count != 1)
{
var numFound = (int)content.data[key].numFound;
if (numFound == count)
{
break;
}
count = numFound;
continue;
}
break;
}
if (isContinue)
{
continue;
}
summary = content.data[key].results[(int)content.data[key].results.Count - 1];
if (!business.IsDownloadByDeliveryId((long)summary.id, key))
{
business.DownloadResumes(cookieContainer, pageNumber - 1, key, cookie.Key, business.GetResumes, token);
continue;
}
#region 中分法查找新纪录的位置
begin = 0;
end = pageNumber - 1;
while (begin <= end)
{
//Console.WriteLine($"Begin：{begin}，End：{end}");
Thread.Sleep(10);
var current = (begin + end) >> 1;
if (begin == end)
{
business.DownloadResumes(cookieContainer, current, key, cookie.Key, business.GetResumes, token);
break;
}
requestResult = business.GetResumes(cookieContainer, current * 90, 90, key, token);
if (!requestResult.IsSuccess)
{
break;
}
content = JsonConvert.DeserializeObject <dynamic>(requestResult.Data);
if ((int)content.code != 1)
{
LogFactory.Warn($"简历列表下载异常！message：{content.message} companyId：{cookie.Key}");
if ((int)content.code == 6001) // 登录过期
{
isBreak = true;
using (var db = new MangningXssDBEntities())
{
var staff = db.ZhaopinStaff.FirstOrDefault(f => f.CompanyId == cookie.Key);
if (staff != null)
{
staff.Cookie = null;
db.SaveChanges();
}
}
break;
}
continue;
}
count = (int)content.data[key].numFound;
if (pageNumber != (int)Math.Ceiling(count / 90.0))
{
pageNumber = (int)Math.Ceiling(count / 90.0);
end = pageNumber - 1;
continue;
}
var summaries = content.data[key].results;
var size = summaries.Count;
// 如果当前页的第一条记录是否下载过，下载过则向前搜索,
// 否则继续判断最后一条是否下载过，如下载过，则断点在
// 当前页,否则向后搜索
if (business.IsDownloadByDeliveryId((long)summaries[0].id, key))
{
end = current;
}
else
{
if (business.IsDownloadByDeliveryId((long)summaries[size - 1].id, key))
{
business.DownloadResumes(cookieContainer, current, key, cookie.Key, business.GetResumes, token);
break;
}
begin = current + 1;
}
}
#endregion
}
#endregion
if (isBreak)
{
continue;
}
#region 载回收站的简历
requestResult = business.GetRecycleResumes(cookieContainer, 0, 1, "recycle", token);
if (!requestResult.IsSuccess)
{
break;
}
content = JsonConvert.DeserializeObject <dynamic>(requestResult.Data);
count = (int)content.data.numFound;
//Console.WriteLine($"CompanyId：{cookie.Key}，分类：Recycle 简历数：{count}");
if (count == 0)
{
continue;
}
if (business.IsDownloadByDeliveryId((long)content.data.results[0].id, "recycle"))
{
continue;
}
pageNumber = (int)Math.Ceiling(count / 90.0);
while (true)
{
requestResult = business.GetRecycleResumes(cookieContainer, count - 1, 90, "recycle", token);
if (!requestResult.IsSuccess)
{
break;
}
content = JsonConvert.DeserializeObject <dynamic>(requestResult.Data);
if ((int)content.data.results.Count != 1)
{
var numFound = (int)content.data.numFound;
if (numFound == count)
{
break;
}
count = numFound;
continue;
}
break;
}
summary = content.data.results[(int)content.data.results.Count - 1];
if (!business.IsDownloadByDeliveryId((long)summary.id, "recycle"))
{
business.DownloadResumes(cookieContainer, pageNumber - 1, "recycle", cookie.Key, business.GetRecycleResumes, token);
continue;
}
#region 中分法查找新纪录的位置
begin = 0;
end = pageNumber - 1;
while (begin <= end)
{
//Console.WriteLine($"Begin：{begin}，End：{end}");
Thread.Sleep(10);
var current = (begin + end) >> 1;
if (begin == end)
{
business.DownloadResumes(cookieContainer, current, "recycle", cookie.Key, business.GetRecycleResumes, token);
break;
}
requestResult = business.GetRecycleResumes(cookieContainer, current * 90, 90, "recycle", token);
if (!requestResult.IsSuccess)
{
break;
}
content = JsonConvert.DeserializeObject <dynamic>(requestResult.Data);
count = (int)content.data.numFound;
if (pageNumber != (int)Math.Ceiling(count / 90.0))
{
pageNumber = (int)Math.Ceiling(count / 90.0);
end = pageNumber - 1;
continue;
}
var summaries = content.data.results;
var size = summaries.Count;
// 如果当前页的第一条记录是否下载过，下载过则向前搜索,
// 否则继续判断最后一条是否下载过，如下载过，则断点在
// 当前页,否则向后搜索
if (business.IsDownloadByDeliveryId((long)summaries[0].id, "recycle"))
{
end = current;
}
else
{
if (business.IsDownloadByDeliveryId((long)summaries[size - 1].id, "recycle"))
{
business.DownloadResumes(cookieContainer, current, "recycle", cookie.Key, business.GetRecycleResumes, token);
break;
}
begin = current + 1;
}
}
#endregion
#endregion
while (true)
{
if (SpiderResumeBusiness.downQueue.Count != 0)
{
Thread.Sleep(1000);
continue;
}
break;
}
}
catch (Exception ex)
{
while (true)
{
if (ex.InnerException == null)
{
break;
}
ex = ex.InnerException;
}
LogFactory.Warn($"下载简历异常！ CompanyId：{cookie.Key} 跳过该 Cookie, 异常信息：{ex.Message} 堆栈：{ex.StackTrace}");
}
}
}));
}
Task.WaitAll(tasks.ToArray());
}
/// <summary>
/// 下载简历
/// </summary>
private static void DownloadResume()
{
while (true)
{
ZhaopinStaff staff;
if (!staffQueue.TryDequeue(out staff))
{
Thread.Sleep(10 * 1000);
continue;
}
try
{
if (!dictionary.TryAdd(staff.Id, staff.CompanyId))
{
continue;
}
var cookieContainer = staff.Cookie.Serialize(".zhaopin.com");
var isWhile = true;
if (staff.Source.Contains("5.5"))
{
#region 网聘 5.5
while (isWhile)
{
var param = new { S_ResumeState = "1", S_CreateDate = $"{DateTime.Now.AddDays(-90):yyMMdd},{DateTime.Now:yyMMdd}", S_feedback = "", page = 1, pageSize = 100 };
var requestResult = RequestFactory.QueryRequest("https://rdapi.zhaopin.com/rd/resume/list", JsonConvert.SerializeObject(param), RequestEnum.POST, cookieContainer, contentType: ContentTypeEnum.Json.Description());
if (!requestResult.IsSuccess)
{
Trace.TraceWarning(requestResult.ErrorMsg);
continue;
}
var content = JsonConvert.DeserializeObject <dynamic>(requestResult.Data);
if ((int)content.code == 4) // 登录过期
{
using (var db = new MangningXssDBEntities())
{
var zhaopinStaff = db.ZhaopinStaff.FirstOrDefault(f => f.Id == staff.Id);
if (zhaopinStaff != null)
{
zhaopinStaff.Cookie = null;
db.SaveChanges();
dictionary.TryRemove(staff.Id, out companyId);
}
}
Trace.WriteLine($"{DateTime.Now} > Loging Timeout ! Username = {staff.Username}, Message = {content.message}.");
break;
}
if ((int)content.code != 0)
{
Trace.WriteLine($"{DateTime.Now} > Get Resume List Error ! Username = {staff.Username}, Message = {content.message}.");
continue;
}
var resumes = content.data.dataList;
var unhandled = (int)content.data.total;
if (resumes.Count == 0)
{
break;
}
HandleResumes(resumes, unhandled, staff, ref isWhile, cookieContainer);
}
#endregion
}
else
{
#region 新系統
foreach (var orderFlag in new[] { "deal", "commu", "interview" })
{
while (isWhile)
{
var requestResult = GetResumes(cookieContainer, 0, 60, orderFlag);
if (!requestResult.IsSuccess)
{
Trace.TraceWarning(requestResult.ErrorMsg);
continue;
}
var content = JsonConvert.DeserializeObject <dynamic>(requestResult.Data);
if ((int)content.code == 6001) // 登录过期
{
using (var db = new MangningXssDBEntities())
{
var zhaopinStaff = db.ZhaopinStaff.FirstOrDefault(f => f.Id == staff.Id);
if (zhaopinStaff != null)
{
zhaopinStaff.Cookie = null;
db.SaveChanges();
dictionary.TryRemove(staff.Id, out companyId);
}
}
Trace.WriteLine($"{DateTime.Now} > Loging Timeout ! Username = {staff.Username}, Message = {content.message}.");
isWhile = false;
break;
}
if ((int)content.code != 1)
{
Trace.WriteLine($"{DateTime.Now} > Get Resume List Error ! Username = {staff.Username}, Message = {content.message}.");
continue;
}
var resumes = content.data[orderFlag].results;
var unhandled = (int)content.data[orderFlag].numFound;
if (resumes.Count == 0)
{
break;
}
HandleResumes(resumes, unhandled, staff, ref isWhile, cookieContainer);
}
}
#endregion
}
}
catch (Exception ex)
{
Trace.TraceError(ex.ToString());
}
finally
{
dictionary.TryRemove(staff.Id, out companyId);
}
}
}
/// <summary>
/// 搜索旧库
/// </summary>
/// <param name="list"></param>
/// <returns></returns>
public static List <ResumeMatchResult> SearchOldLibrary(IEnumerable <dynamic> list)
{
var resumeList = new List <ResumeMatchResult>();
using (var db = new ResumeMatchDBEntities())
{
foreach (var item in list)
{
var resumeNumber = ((string)item.ResumeNumber).Substring(0, 10);
var resume = db.OldResumeSummary.FirstOrDefault(w => w.ResumeId == resumeNumber);
if (resume == null)
{
continue;
}
resume.MatchTime = DateTime.Now;
resume.IsMatched = true;
resumeList.Add(new ResumeMatchResult
{
Cellphone = resume.Cellphone,
Email = resume.Email,
ResumeNumber = item.ResumeNumber,
Status = 2
});
using (var xdb = new MangningXssDBEntities())
{
var filePath = "X:" + ((string)item.SavePath).Replace("/", "\\");
if (!File.Exists(filePath))
{
LogFactory.Warn($"找不到文件路径：{filePath}, ResumeNumber：{resumeNumber}");
continue;
}
var resumeJson = JsonConvert.DeserializeObject <dynamic>(File.ReadAllText(filePath));
var detail = JsonConvert.DeserializeObject(resumeJson.detialJSonStr.ToString());
var resumeId = resumeJson.resumeId != null ? (int)resumeJson.resumeId : detail.ResumeId != null ? (int)detail.ResumeId : 0;
if (resumeId == 0)
{
LogFactory.Warn($"解析异常！ResumeId 为空, ResumeNumber：{resumeNumber}");
continue;
}
var userId = (int)resumeJson.userDetials.userMasterId;
var user = xdb.ZhaopinUser.FirstOrDefault(f => f.Id == userId);
if (user != null)
{
if (!user.Source.Contains("MANUAL"))
{
user.Id = userId;
user.Source = "XSS";
user.ModifyTime = BaseFanctory.GetTime((string)detail.DateModified).ToUniversalTime();
user.CreateTime = BaseFanctory.GetTime((string)detail.DateCreated).ToUniversalTime();
user.Cellphone = resume.Cellphone;
user.Email = resume.Email;
user.Name = resumeJson.userDetials.userName.ToString();
user.UpdateTime = DateTime.UtcNow;
user.Username = resumeJson.userDetials.email.ToString();
}
}
else
{
xdb.ZhaopinUser.AddOrUpdate(a => a.Id, new ZhaopinUser
{
Id = userId,
Source = "XSS",
ModifyTime = BaseFanctory.GetTime((string)detail.DateModified).ToUniversalTime(),
CreateTime = BaseFanctory.GetTime((string)detail.DateCreated).ToUniversalTime(),
Cellphone = resume.Cellphone,
Email = resume.Email,
Name = resumeJson.userDetials.userName.ToString(),
UpdateTime = DateTime.UtcNow,
Username = resumeJson.userDetials.email.ToString()
});
}
var resumeEntity = xdb.ZhaopinResume.FirstOrDefault(f => f.Id == resumeId);
if (resumeEntity == null)
{
xdb.ZhaopinResume.Add(new ZhaopinResume
{
Id = resumeId,
RandomNumber = resumeNumber,
UserId = userId,
RefreshTime = BaseFanctory.GetTime((string)detail.DateModified).ToUniversalTime(),
UpdateTime = DateTime.UtcNow,
UserExtId = detail.UserMasterExtId.ToString(),
DeliveryNumber = null,
Source = "XSS"
});
}
else
{
resumeEntity.Id = resumeId;
resumeEntity.RandomNumber = resumeNumber;
resumeEntity.UserId = userId;
resumeEntity.RefreshTime = BaseFanctory.GetTime((string)detail.DateModified).ToUniversalTime();
resumeEntity.UpdateTime = DateTime.UtcNow;
resumeEntity.UserExtId = detail.UserMasterExtId.ToString();
resumeEntity.DeliveryNumber = resumeEntity.DeliveryNumber;
resumeEntity.Source = resumeEntity.Source;
}
xdb.SaveChanges();
var path = $"{ConfigurationManager.AppSettings["Resume.SavePath"]}{DateTime.Now:yyyyMMdd}";
if (!Directory.Exists(path))
{
Directory.CreateDirectory(path);
}
var resumePath = $@"{path}\{resumeId}.json";
File.WriteAllText(resumePath, JsonConvert.SerializeObject(resumeJson));
uploadOssActionBlock.Post(resumePath);
}
}
db.TransactionSaveChanges();
}
return(resumeList);
}
/// <summary>
/// 标记简历
/// </summary>
/// <param name="jsonContent"></param>
/// <param name="resumeId"></param>
/// <param name="client"></param>
/// <param name="bucketName"></param>
/// <returns></returns>
private static short FlagResume(string jsonContent, int resumeId, IOss client, string bucketName)
{
using (var db = new MangningXssDBEntities())
{
var resume = db.ZhaopinResume.FirstOrDefault(f => f.Id == resumeId);
if (resume == null)
{
return(0x0);
}
if (jsonContent.Contains("detialJSonStr"))
{
var jsonObj = JsonConvert.DeserializeObject <dynamic>(jsonContent);
if (!string.IsNullOrWhiteSpace((string)jsonObj.userDetials.mobilePhone))
{
resume.Flag = 0xF;
}
else
{
var user = db.ZhaopinUser.FirstOrDefault(f => f.Id == resume.UserId && !string.IsNullOrEmpty(f.Cellphone));
if (user == null)
{
resume.Flag = 0xD;
}
else
{
jsonObj.userDetials.mobilePhone = user.Cellphone;
jsonObj.userDetials.email = user.Email;
resume.Flag = 0xF;
var jsonResume = JsonConvert.SerializeObject(jsonObj);
try
{
using (var stream = new MemoryStream(GZip.Compress(Encoding.UTF8.GetBytes(jsonResume))))
{
client.PutObject(bucketName, $"Zhaopin/Resume/{resumeId}", stream);
}
//if (resume.Flag != 0xD) File.WriteAllText($@"F:\ZhaopinOss\Resume\{resumeId}", jsonResume);
}
catch (Exception ex)
{
Trace.TraceError(ex.ToString());
}
}
}
}
else
{
resume.Flag = 0x9;
}
db.SaveChanges();
return(resume.Flag);
}
}
public void Match()
{
if (!Directory.Exists(savePath))
{
Directory.CreateDirectory(savePath);
}
var cookieQueue = new ConcurrentQueue <KeyValuePair <KeyValuePair <int, string>, CookieContainer> >();
var companyDic = new ConcurrentDictionary <string, int>();
//using (var db = new MangningXssDBEntities())
//{
// var companyArr = db.ZhaoPinCompany.Where(w => w.Source.Contains("MANUAL")).Select(s => s.Id).ToArray();
// var paramArr = db.ZhaopinStaff.Where(w => companyArr.Any(a => a == w.CompanyId) && !string.IsNullOrEmpty(w.Cookie)).Select(s => new { s.CompanyId, s.Username, s.Cookie }).ToArray();
// foreach (var item in paramArr)
// {
// companyDic[item.Username] = 0;
// cookieQueue.Enqueue(new KeyValuePair<KeyValuePair<int, string>, CookieContainer>(new KeyValuePair<int, string>(item.CompanyId, item.Username), item.Cookie.Serialize(".zhaopin.com")));
// }
//}
using (var db = new MangningXssDBEntities())
{
var list = db.ZhaopinCleaningProcedure.Where(w => w.IsEnable /* && !w.IsOnline*/ && !string.IsNullOrEmpty(w.Cookie));
foreach (var item in list)
{
if (item.StartTime < DateTime.Today)
{
item.TodayWatch = 0;
}
companyDic[item.Account] = item.TodayWatch;
cookieQueue.Enqueue(new KeyValuePair <KeyValuePair <int, string>, CookieContainer>(new KeyValuePair <int, string>(0, item.Account), item.Cookie.Serialize(".zhaopin.com")));
item.IsOnline = true;
item.StartTime = DateTime.Now;
}
db.SaveChanges();
}
Console.WriteLine($"已获取 Cookie 数 => {cookieQueue.Count}");
Task.Run(() =>
{
while (true)
{
if (searchQueue.Count > 500)
{
Thread.Sleep(100);
continue;
}
var list = this.GetOldResumes();
foreach (var item in list)
{
searchQueue.Enqueue(item);
}
}
});
for (var j = 0; j < 16; j++)
{
Task.Run(() =>
{
try
{
while (true)
{
KeyValuePair <KeyValuePair <int, string>, CookieContainer> temp;
if (!cookieQueue.TryDequeue(out temp))
{
Thread.Sleep(1000);
Console.WriteLine($"Company => {temp.Key.Value} 找不到可用Cookie！");
continue;
}
while (true)
{
SearchResumeModel search;
if (!searchQueue.TryDequeue(out search))
{
Console.WriteLine($"Company => {temp.Key.Value} 找不到可搜索的条件！");
Thread.Sleep(1000);
continue;
}
var pageIndex = 1;
var pageTotal = 1;
var total = 0;
var isbreak = false;
var gender = search.Gender == "男" ? 1 : 2;
var isMatched = false;
foreach (var company in search.Companys)
{
while (pageIndex < pageTotal + 1)
{
var url = $"https://ihrsearch.zhaopin.com/Home/ResultForCustom?SF_1_1_9={gender}&SF_1_1_25=COMPANY_NAME_ALL:{HttpUtility.UrlEncode(company)}&orderBy=BIRTH_YEAR%2C0&SF_1_1_27=0&exclude=1&pageIndex={pageIndex}&pageSize=60";
var requestResult = RequestFactory.QueryRequest(url, cookieContainer: temp.Value, referer: url);
if (!requestResult.IsSuccess)
{
LogFactory.Warn($"Company => {temp.Key.Value} 条件搜索异常！异常原因=>{requestResult.ErrorMsg} Condition=>{JsonConvert.SerializeObject(search)}");
continue;
}
if (pageTotal == 1)
{
var match = Regex.Match(requestResult.Data, "(?s)<span>(\\d+)</span>份简历.+?rd-resumelist-pageNum\">1/(\\d+)</span>");
if (!match.Success)
{
if (requestResult.Data.Contains("text/javascript\" r='m'"))
{
LogFactory.Warn($"Cookie 过期，过期用户 => {temp.Key.Value}");
RemoveCookie(temp.Key.Value, companyDic[temp.Key.Value]);
}
else
{
LogFactory.Warn($"Company => {temp.Key.Value} 条件搜索异常！返回页面解析异常！{requestResult.Data}");
}
isbreak = true;
break;
}
total = Convert.ToInt32(match.Result("$1"));
pageTotal = Convert.ToInt32(match.Result("$2"));
if (total == 0)
{
Console.WriteLine("该条件无搜索结果！");
break;
}
}
Console.WriteLine($"Company => {temp.Key.Value} 第 {pageIndex} 页 共 {pageTotal} 页 {total} 个结果");
if (pageTotal > 1)
{
break;
}
pageIndex++;
var matchs = Regex.Matches(requestResult.Data, "(?s)RedirectToRd/([^\r\n]+?)','([^\r\n]+?)','([^\r\n]+?)',this\\);this.+?(\\d{2}-\\d{2}-\\d{2})");
var index = 0;
foreach (Match item in matchs)
{
var number = item.Result("$1").Substring(0, 10);
var numberParam = item.Result("$1").Substring(0, item.Result("$1").IndexOf("?", StringComparison.Ordinal));
var cache = GetResumeOfCache(number);
if (cache == null)
{
DateTime updateDateTime;
if (DateTime.TryParse(item.Result("$4"), out updateDateTime))
{
QueryResumeIsExists(number, updateDateTime);
}
requestResult = RequestFactory.QueryRequest($"http://ihr.zhaopin.com/resumesearch/getresumedetial.do?resumeNo={numberParam}&searchresume=1&resumeSource=1&keyword=%E8%AF%B7%E8%BE%93%E5%85%A5%E7%AE%80%E5%8E%86%E5%85%B3%E9%94%AE%E8%AF%8D%EF%BC%8C%E5%A4%9A%E5%85%B3%E9%94%AE%E8%AF%8D%E5%8F%AF%E7%94%A8%E7%A9%BA%E6%A0%BC%E5%88%86%E9%9A%94&t={item.Result("$2")}&k={item.Result("$3")}&v=undefined&version=3&openFrom=1", cookieContainer: temp.Value);
if (!requestResult.IsSuccess)
{
LogFactory.Warn($"Company => {temp.Key.Value} 简历详情查看异常！异常原因=>{requestResult.ErrorMsg} ResumeNumber=>{number}");
continue;
}
try
{
var jsonObj = JsonConvert.DeserializeObject <dynamic>(requestResult.Data);
if ((int)jsonObj["code"] != 1)
{
LogFactory.Warn($"Company => {temp.Key.Value} ResumeNumber => {number} 查看详情异常 信息：{(string)jsonObj["message"]} 查看简历份数：{companyDic[temp.Key.Value]}");
if (((string)jsonObj["message"]).Contains("当日查看简历已达上限"))
{
RemoveCookie(temp.Key.Value, companyDic[temp.Key.Value]);
isbreak = true;
break;
}
continue;
}
var resumeData = jsonObj.data;
var userId = (string)resumeData.userDetials.userMasterId;
if (search.SearchResumeId.Substring(2, 8) == (userId.Length == 8 ? userId : userId.Substring(1)))
{
Console.WriteLine($"匹配简历成功！ResumeNumber=>{number} 今日匹配成功数=> {++totalMatched}");
resumeData.userDetials.mobilePhone = search.Cellphone;
resumeData.userDetials.email = search.Email;
isMatched = true;
}
SaveCacheByOss(new KeyValuePair <KeyValuePair <int, string>, dynamic>(temp.Key, resumeData));
Console.WriteLine($"Company => {temp.Key.Value} 查看简历成功！查看简历份数：{++companyDic[temp.Key.Value]} ResumeNumber => {number} {++index}/{matchs.Count}");
if (isMatched)
{
break;
}
}
catch (Exception ex)
{
while (true)
{
if (ex.InnerException == null)
{
break;
}
ex = ex.InnerException;
}
LogFactory.Error($"保存简历信息异常！异常信息：{ex.Message} 响应信息：{requestResult.Data}");
}
}
else
{
var userId = cache.UserId.ToString();
if (search.SearchResumeId.Substring(2, 8) == (userId.Length == 8 ? userId : userId.Substring(1)))
{
Console.WriteLine($"匹配简历成功！ResumeNumber=>{number} 今日匹配成功数=> {++totalMatched}");
using (var xdb = new MangningXssDBEntities())
{
var model = xdb.ZhaopinMatchedCache.FirstOrDefault(f => f.ResumeId == cache.ResumeId);
if (model != null)
{
model.Cellphone = search.Cellphone;
model.Email = search.Email;
xdb.SaveChanges();
}
}
isMatched = true;
break;
}
}
}
if (isbreak || isMatched)
{
break;
}
}
if (isbreak || isMatched)
{
break;
}
}
ChangeResumeStatus(search.SearchResumeId, isMatched);
if (isbreak)
{
break;
}
}
}
}
catch (Exception ex)
{
while (true)
{
if (ex.InnerException == null)
{
break;
}
ex = ex.InnerException;
}
LogFactory.Error($"Watch异常！异常信息：{ex.Message} {ex.StackTrace}");
}
});
}
}
/// <summary>
/// 下载
/// </summary>
/// <param name="cookieContainer"></param>
/// <param name="jobNumber"></param>
/// <param name="number"></param>
/// <param name="refreshTime"></param>
/// <param name="orderFlag"></param>
/// <param name="deliveryId"></param>
/// <param name="companyId"></param>
/// <param name="token"></param>
private static void Down(CookieContainer cookieContainer, string jobNumber, string number, DateTime refreshTime, string orderFlag, long deliveryId, int companyId, string token)
{
var resumeId = 0;
var url = orderFlag == "recycle" ? $"http://ihr.zhaopin.com/resumesearch/getresumedetial.do?access_token={token}&resumeJobId={deliveryId}" : $"http://ihr.zhaopin.com/resumesearch/getresumedetial.do?resumeNo={deliveryId}_{jobNumber}_{number}_1_1&resumeSource=3";
try
{
var requestResult = RequestFactory.QueryRequest(url, cookieContainer: cookieContainer);
if (!requestResult.IsSuccess)
{
LogFactory.Warn($"简历信息下载异常！异常信息：{requestResult.ErrorMsg}，CompanyId：{companyId}，jobNumber：{jobNumber}, ResumeNumber:{number} ");
return;
}
var resume = JsonConvert.DeserializeObject <dynamic>(requestResult.Data);
if ((int)resume.code != 1)
{
LogFactory.Warn($"简历信息下载异常！message：{resume.message}，CompanyId：{companyId}，jobNumber：{jobNumber}, ResumeNumber:{number} ");
if ((int)resume.code == 6001)
{
while (true)
{
Tuple <CookieContainer, string, string, DateTime, string, long, int, Tuple <string> > param;
if (!downQueue.TryDequeue(out param))
{
break;
}
if (param.Item7 != companyId)
{
downQueue.Enqueue(param);
}
}
return; // 登录过期
}
return;
}
Thread.Sleep(1000);
if (resume.data == null)
{
return;
}
var detail = JsonConvert.DeserializeObject(resume.data.detialJSonStr.ToString());
resume.data.detialJSonStr = detail;
var path = $"{ConfigurationManager.AppSettings["Resume.SavePath"]}";
if (!Directory.Exists(path))
{
Directory.CreateDirectory(path);
}
resumeId = (int)resume.data.resumeId;
using (var db = new MangningXssDBEntities())
{
if (db.ZhaopinDelivery.Any(a => a.Id == deliveryId))
{
Console.WriteLine($"DeliveryId：{deliveryId} 已下载过！");
return;
}
db.ZhaopinDelivery.Add(new ZhaopinDelivery
{
CompanyId = companyId,
Id = deliveryId,
ResumeId = resumeId,
JobNumber = jobNumber,
ResumeNumber = number
});
if (!db.ZhaopinDeliveryLog.Any(a => a.DeliveryId == deliveryId))
{
db.ZhaopinDeliveryLog.Add(new ZhaopinDeliveryLog
{
DeliveryId = deliveryId,
CompanyId = companyId
});
}
db.TransactionSaveChanges();
if (db.ZhaopinResume.Any(a => a.Id == resumeId && a.RefreshTime >= refreshTime && a.Flag != 2))
{
var resumeNumber = number.Substring(0, 10);
Console.WriteLine($"简历已下载过！ResumeNumber：{resumeNumber}");
return;
}
}
if (!resumeDic.TryAdd(resumeId, resumeId))
{
return;
}
var resumePath = $@"{path}\{resumeId}.json";
File.WriteAllText(resumePath, JsonConvert.SerializeObject(resume.data));
uploadOssActionBlock.Post(resumePath);
var userId = (int)resume.data.userDetials.userMasterId;
using (var db = new MangningXssDBEntities())
{
var user = db.ZhaopinUser.FirstOrDefault(f => f.Id == userId);
if (user != null)
{
if (!user.Source.Contains("MANUAL"))
{
user.Id = userId;
user.Source = "XSS";
user.ModifyTime = BaseFanctory.GetTime((string)detail.DateModified).ToUniversalTime();
user.CreateTime = BaseFanctory.GetTime((string)detail.DateCreated).ToUniversalTime();
user.Cellphone = resume.data.userDetials.mobilePhone.ToString();
user.Email = resume.data.userDetials.email.ToString();
user.Name = resume.data.userDetials.userName.ToString();
user.UpdateTime = DateTime.UtcNow;
user.Username = resume.data.userDetials.email.ToString();
}
}
else
{
db.ZhaopinUser.AddOrUpdate(a => a.Id, new ZhaopinUser
{
Id = userId,
Source = "XSS",
ModifyTime = BaseFanctory.GetTime((string)detail.DateModified).ToUniversalTime(),
CreateTime = BaseFanctory.GetTime((string)detail.DateCreated).ToUniversalTime(),
Cellphone = resume.data.userDetials.mobilePhone.ToString(),
Email = resume.data.userDetials.email.ToString(),
Name = resume.data.userDetials.userName.ToString(),
UpdateTime = DateTime.UtcNow,
Username = resume.data.userDetials.email.ToString()
});
}
var resumeEntity = db.ZhaopinResume.FirstOrDefault(f => f.Id == resumeId);
var userExdId = Regex.IsMatch(detail.UserMasterExtId.ToString(), @"^J[MRSL]\d{9}$") ? detail.UserMasterExtId.ToString() : string.Empty;
if (resumeEntity == null)
{
db.ZhaopinResume.Add(new ZhaopinResume
{
Id = resumeId,
RandomNumber = number.Substring(0, 10),
UserId = userId,
RefreshTime = BaseFanctory.GetTime((string)detail.DateModified).ToUniversalTime(),
UpdateTime = DateTime.UtcNow,
UserExtId = userExdId,
DeliveryNumber = null,
Source = "XSS,Deliver",
Flag = 0xE
});
}
else
{
resumeEntity.RandomNumber = number.Substring(0, 10);
resumeEntity.UserId = userId;
resumeEntity.RefreshTime = BaseFanctory.GetTime((string)detail.DateModified).ToUniversalTime();
resumeEntity.UpdateTime = DateTime.UtcNow;
if (string.IsNullOrEmpty(resumeEntity.UserExtId))
{
resumeEntity.UserExtId = userExdId;
}
resumeEntity.DeliveryNumber = resumeEntity.DeliveryNumber;
resumeEntity.Source = !resumeEntity.Source.Contains("Deliver") ? resumeEntity.Source += ",Deliver" : resumeEntity.Source;
resumeEntity.Flag = 0xE;
if (resumeEntity.IncludeTime == null)
{
resumeEntity.IncludeTime = DateTime.UtcNow;
}
}
db.SaveChanges();
}
Interlocked.Increment(ref downloadCount);
resumeDic.TryRemove(resumeId, out resumeId);
Console.WriteLine($"下载成功！{downloadCount}/{downQueue.Count} CId:{companyId} RId:{resumeId} DId:{deliveryId} orderFlag:{orderFlag} oss:{uploadOssActionBlock.InputCount}");
}
catch (Exception ex)
{
while (true)
{
if (ex.InnerException == null)
{
break;
}
ex = ex.InnerException;
}
LogFactory.Warn($"下载简历异常！异常信息：{ex.Message} 堆栈：{ex.StackTrace} CompanyId：{companyId}，ResumeId:{resumeId}, orderFlag：{orderFlag}");
}
}
public void MatchResume()
{
const string path = @"D:\待清理数据\2017-11-24 智联招聘简历导出";
var filesPath = Directory.EnumerateFileSystemEntries(path);
var dictionary = new ConcurrentDictionary <string, string>();
var index = 0;
var queue = new ConcurrentQueue <string>();
foreach (var filePath in filesPath)
{
queue.Enqueue(filePath);
}
for (var i = 0; i < 8; i++)
{
Task.Run(() =>
{
while (true)
{
string filePath;
if (!queue.TryDequeue(out filePath))
{
continue;
}
Interlocked.Increment(ref index);
var htmlSource = File.ReadAllText(filePath);
var numberMatchResult = Regex.Match(htmlSource, "tips-id\">ID:(\\S{10})");
if (!numberMatchResult.Success)
{
LogFactory.Warn("简历编号匹配失败！Path：" + filePath);
continue;
}
var mobileMatchResult = Regex.Match(htmlSource, "main-title-fr\">(Mobile|手机) ：.*?(\\d{11})");
if (!mobileMatchResult.Success)
{
LogFactory.Warn("手机号码匹配失败！Path：" + filePath);
continue;
}
var resumeNumber = numberMatchResult.Result("$1").Trim().Substring(0, 10);
var mobile = mobileMatchResult.Result("$2");
using (var db = new MangningXssDBEntities())
{
if (db.ZhaopinResume.Any(a => a.RandomNumber.StartsWith(resumeNumber)))
{
dictionary.TryAdd(resumeNumber, mobile);
continue;
}
if (db.ZhaopinUser.Any(a => a.Cellphone == mobile))
{
dictionary.TryAdd(resumeNumber, mobile);
continue;
}
}
using (var db = new BadoucaiAliyunDBEntities())
{
if (db.CoreResumeReferenceMapping.Any(a => a.Value.StartsWith(resumeNumber)))
{
dictionary.TryAdd(resumeNumber, mobile);
continue;
}
if (db.CoreResumeSummary.Any(a => a.Cellphone.ToString() == mobile))
{
dictionary.TryAdd(resumeNumber, mobile);
}
}
}
});
}
SpinWait.SpinUntil(() => false);
}
/// <summary>
/// 获取 Cookie 相关信息
/// </summary>
/// <returns></returns>
public Dictionary <int, string> GetCookies()
{
dictionary.Clear();
var cookieList = new Dictionary <int, string>();
var cookieStaffList = new List <ZhaopinStaff>();
using (var db = new MangningXssDBEntities())
{
while (true)
{
try
{
var pageIndex = 0;
while (true)
{
//var list = db.ZhaopinStaff.Where(w => !string.IsNullOrEmpty(w.Cookie)).OrderByDescending(o => o.Id).Skip(pageIndex * 500).Take(500).ToList();
var companyArr = db.ZhaoPinCompany.Where(w => w.Source.Contains("MANUAL")).Select(s => s.Id).ToArray();
var list = db.ZhaopinStaff.Where(w => companyArr.Any(a => a == w.CompanyId) && !string.IsNullOrEmpty(w.Cookie)).OrderByDescending(o => o.Id).Skip(pageIndex * 500).Take(500).ToList();
if (!list.Any())
{
break;
}
cookieStaffList.AddRange(list);
pageIndex++;
}
cookieStaffList = cookieStaffList.Distinct().ToList();
break;
}
catch (Exception ex)
{
Console.WriteLine($"获取 Cookie 异常！{ex.Message} 准备重试...");
}
}
}
foreach (var cookie in cookieStaffList)
{
if (dictionary.ContainsKey(cookie.Id))
{
if (cookie.UpdateTime?.Subtract(dictionary[cookie.Id]).TotalMinutes < 30)
{
continue;
}
dictionary[cookie.Id] = cookie.UpdateTime ?? DateTime.Now;
}
else
{
dictionary.Add(cookie.Id, cookie.UpdateTime ?? DateTime.Now);
}
cookieList.Add(cookie.CompanyId, cookie.Cookie);
}
Console.WriteLine($"获取 Cookie 完成，共 {cookieStaffList.Count} 个，{cookieList.Count} 个待爬取！");
return(cookieList);
}
/// <summary>
/// 处理简历
/// </summary>
/// <param name="item"></param>
/// <param name="cellphone"></param>
/// <param name="email"></param>
/// <param name="cleaningProcedure"></param>
/// <param name="businessId"></param>
private static void HandleResume(Match item, string cellphone, string email, ZhaopinCleaningProcedure cleaningProcedure, int businessId)
{
var number = item.Result("$1").Substring(0, 10);
DateTime updateDateTime;
if (!DateTime.TryParse(item.Result("$4"), out updateDateTime))
{
return;
}
using (var db = new MangningXssDBEntities())
{
var resume = db.ZhaopinResume.FirstOrDefault(f => f.RandomNumber == number);
if (resume != null)
{
if (resume.Flag == 15)
{
#region 刷新简历更新时间
//if (resume.RefreshTime != null && updateDateTime.Date == resume.RefreshTime.Value.Date) continue;
resume.RefreshTime = updateDateTime;
resume.Flag = 14;
var filePath = $@"D:\Badoucai\Resume\LocationJson\{resume.Id}.json";
using (var stream = new MemoryStream())
{
if (!mangningOssClient.DoesObjectExist(mangningBucketName, $"Zhaopin/Resume/{resume.Id}"))
{
return;
}
var bytes = new byte[1024];
int len;
var streamContent = mangningOssClient.GetObject(mangningBucketName, $"Zhaopin/Resume/{resume.Id}").Content;
while ((len = streamContent.Read(bytes, 0, bytes.Length)) > 0)
{
stream.Write(bytes, 0, len);
}
var resumeContent = Encoding.UTF8.GetString(GZip.Decompress(stream.ToArray()));
var jsonObj = JsonConvert.DeserializeObject <dynamic>(resumeContent);
dynamic detialJSonStr;
try
{
detialJSonStr = jsonObj.detialJSonStr;
if (!string.IsNullOrEmpty((string)jsonObj.detialJSonStr.DateModified))
{
jsonObj.detialJSonStr = JsonConvert.SerializeObject(jsonObj.detialJSonStr);
}
}
catch (Exception)
{
detialJSonStr = JsonConvert.DeserializeObject <dynamic>((string)jsonObj.detialJSonStr);
}
detialJSonStr.DateLastReleased = updateDateTime;
detialJSonStr.DateModified = updateDateTime;
jsonObj.detialJSonStr = detialJSonStr;
var newResumeContent = JsonConvert.SerializeObject(jsonObj);
using (var jsonStream = new MemoryStream(GZip.Compress(Encoding.UTF8.GetBytes(newResumeContent))))
{
mangningOssClient.PutObject(mangningBucketName, $"Zhaopin/Resume/{resume.Id}", jsonStream);
}
File.WriteAllText(filePath, newResumeContent);
}
#endregion
}
if (resume.Flag == 2)
{
var user = db.ZhaopinUser.FirstOrDefault(f => f.Id == resume.UserId);
if (user == null)
{
return;
}
if (!WatchResumeDetail(item, cookieContainer, user.Cellphone, user.Email, businessId))
{
var procedure = db.ZhaopinCleaningProcedure.FirstOrDefault(f => f.Id == cleaningProcedure.Id);
if (procedure != null)
{
procedure.Cookie = null;
db.SaveChanges();
}
businessIdQueue.Enqueue(businessId);
}
}
}
else
{
var incompleteResume = db.ZhaopinIncompleteResume.FirstOrDefault(f => f.ResumeNumber == number);
if (incompleteResume != null)
{
incompleteResume.CompletionTime = DateTime.Now;
}
WatchResumeDetail(item, cookieContainer, cellphone, email, businessId);
}
db.SaveChanges();
}
}
/// <summary>
/// 投递简历
/// </summary>
private static void DeilverTask()
{
var userResumeQueue = new Queue <Tuple <dynamic, int> >();
var positionQueue = new Queue <dynamic>();
#region 填充用户简历及职位队列
using (var db = new MangningXssDBEntities())
{
var date = DateTime.Now.Date.AddHours(-8);
var userResumeRecordList = db.ZhaopinDeilveryRecord.AsNoTracking().Where(w => w.CreateTime > date).GroupBy(g => g.UserId).Select(s => s.Key).ToList();
if (userResumeRecordList.Any())
{
var userResumes = db.ZhaopinResume
.Join(db.ZhaopinUser.Where(w => userResumeRecordList.Any(a => a == w.Id) && !string.IsNullOrEmpty(w.Cookie) && w.Source.Contains("MANUAL") && string.IsNullOrEmpty(w.Status)), a => a.UserId, b => b.Id, (a, b) => new { b.Username, a.UserId, ResumeNumber = a.DeliveryNumber, b.Cookie })
.ToList();
foreach (var user in userResumes)
{
if (userResumeQueue.Count >= deilverUserCount)
{
break;
}
var todayDeilverCount = db.ZhaopinDeilveryRecord.Count(w => w.UserId == user.UserId && w.CreateTime > date);
//if (string.IsNullOrEmpty(user.Cookie))
//{
// LogFactory.Warn($"投递用户：{user.Username} Cookie 已过期！请录入有效 Cookie ！");
// return;
//}
userResumeQueue.Enqueue(new Tuple <dynamic, int>(user, todayDeilverCount));
}
}
if (userResumeQueue.Count < deilverUserCount)
{
var userResumeList = db.ZhaopinResume
.Join(db.ZhaopinUser.Where(w => userResumeRecordList.All(a => a != w.Id) && !string.IsNullOrEmpty(w.Cookie) && w.Source.Contains("MANUAL") && string.IsNullOrEmpty(w.Status)), a => a.UserId, b => b.Id, (a, b) => new { b.Username, a.UserId, ResumeNumber = a.DeliveryNumber, b.Cookie })
.ToList();
foreach (var f in userResumeList)
{
var todayDeilverCount = db.ZhaopinDeilveryRecord.Count(w => w.UserId == f.UserId && w.CreateTime > date);
if (todayDeilverCount < singleUserDeilverCount)
{
if (userResumeQueue.Count >= deilverUserCount)
{
break;
}
int code;
if (CheckUserIsUseable(f, out code))
{
userResumeQueue.Enqueue(new Tuple <dynamic, int>(f, todayDeilverCount));
}
else
{
var user = db.ZhaopinUser.FirstOrDefault(df => df.Id == f.UserId);
if (user != null)
{
user.Status = "BLOCKED";
if (code == -1)
{
user.Status = "ABNORMAL";
}
user.UpdateTime = DateTime.UtcNow;
db.SaveChanges();
}
}
}
}
}
if (!userResumeQueue.Any())
{
Console.WriteLine("等待可用用户中....");
return;
}
if (userResumeQueue.All(a => a.Item2 >= singleUserDeilverCount))
{
var todayCount = db.ZhaopinDeilveryRecord.AsNoTracking().Count(c => c.CreateTime > date);
if (todayCount < singleUserDeilverCount * deilverUserCount)
{
Console.WriteLine("等待可用用户中....");
return;
}
Console.WriteLine("今日投递任务已完成....");
return;
}
var positionList = db.ZhaopinDeilverTask
.Where(w => w.Status == 0)
.Join(db.ZhaopinPosition, a => a.CompanyId, b => b.CompanyId, (a, b) => new { TaskId = a.Id, a.CompanyId, PositionId = b.Id, b.Number, b.CreateTime, b.IsEnable, b.Name })
.OrderByDescending(o => o.CreateTime)
.ToList();
var deilverTaskList = db.ZhaopinDeilverTask.Where(w => w.Status == 0).OrderBy(o => o.Priority).ToList();
date = DateTime.UtcNow.AddDays(-7);
foreach (var f in deilverTaskList)
{
var positions = positionList
.Where(w => w.TaskId == f.Id && w.IsEnable && w.CreateTime > date)
.OrderByDescending(o => o.CreateTime)
.Skip(f.ActualDeilverCount)
.Take(f.ExpectedDeilverCount - f.ActualDeilverCount)
.ToList();
if (!positions.Any())
{
f.Status = 2;
f.CompleteTime = DateTime.UtcNow;
continue;
}
positions = positions.Take(f.ExpectedDeilverCount - f.ActualDeilverCount).ToList();
positions.ForEach(pf =>
{
positionQueue.Enqueue(pf);
});
}
db.TransactionSaveChanges();
}
if (!positionQueue.Any())
{
Console.WriteLine("等待分配任务中....");
return;
}
#endregion
while (true)
{
if (!positionQueue.Any())
{
return;
}
var position = positionQueue.Dequeue();
if (!userResumeQueue.Any())
{
return;
}
var userResume = userResumeQueue.Dequeue();
#region 投递简历
var userDeilverNumber = 0;
do
{
var deilverResult = RequestFactory.QueryRequest($"https://my.zhaopin.com/v5/FastApply/resumeinfo.aspx?t=3&j={position.Number}&j2=&so=&su=&ff=ssb&rv={userResume.Item1.ResumeNumber}_1&rl=1&cl=&sd=0&fd=&c=jsonp7u3t5v&_=1493174565322", cookieContainer: ((string)userResume.Item1.Cookie).Serialize("zhaopin.com"));
if (!deilverResult.IsSuccess)
{
LogFactory.Warn($"投递简历请求异常！简历编号：{userResume.Item1.ResumeNumber}，职位编号：{position.Number}，异常信息：{deilverResult.ErrorMsg}");
continue;
}
var message = JsonConvert.DeserializeObject <dynamic>(Regex.Match(deilverResult.Data, @"(\{.+?\})").Result("$1"));
if (message.loginstatus.ToString().Contains("7_Position"))
{
LogFactory.Warn($"该职位已下架！职位编号：{position.Number} Json:{JsonConvert.SerializeObject(message)}");
using (var db = new MangningXssDBEntities())
{
var positionId = (int)position.PositionId;
var positionDefault = db.ZhaopinPosition.FirstOrDefault(f => f.Id == positionId);
if (positionDefault != null)
{
positionDefault.IsEnable = false;
}
else
{
LogFactory.Warn("数据库未找到对应的职位！职位ID：" + positionId);
}
db.TransactionSaveChanges();
}
if (!positionQueue.Any())
{
return;
}
position = positionQueue.Dequeue();
continue;
}
if (message.loginstatus.ToString() != "6")
{
LogFactory.Warn($"用户登录 Cookie 失效！用户ID：{userResume.Item1.UserId} Json:{JsonConvert.SerializeObject(message)}");
using (var db = new MangningXssDBEntities())
{
var userId = (int)userResume.Item1.UserId;
var user = db.ZhaopinUser.FirstOrDefault(f => f.Id == userId);
if (user != null)
{
user.Cookie = "";
user.UpdateTime = DateTime.UtcNow;
}
else
{
LogFactory.Warn("数据库未找到对应的用户！用户ID：" + userId);
}
//db.TransactionSaveChanges();
}
if (!userResumeQueue.Any())
{
return;
}
userResume = userResumeQueue.Dequeue();
continue;
}
var status = message.postBackInfo.ToString().Split('_');
if (status[3].ToString() == "0")
{
if (status[0] == "0" && status[1] == "0" && status[2] == "0")
{
using (var db = new MangningXssDBEntities())
{
var userId = (int)userResume.Item1.UserId;
var user = db.ZhaopinUser.FirstOrDefault(f => f.Id == userId);
if (user != null)
{
user.Status = "PENDING";
user.UpdateTime = DateTime.UtcNow;
db.TransactionSaveChanges();
}
}
LogFactory.Warn($"该帐号已不能继续投递！用户ID：{userResume.Item1.UserId}");
if (!userResumeQueue.Any())
{
return;
}
userResume = userResumeQueue.Dequeue();
continue;
}
LogFactory.Warn($"七天内已投递过该职位！用户ID：{userResume.Item1.UserId}，职位编号：{position.Number}，职位名称：{position.Name}");
if (userDeilverNumber < userResumeQueue.Count)
{
userResumeQueue.Enqueue(userResume);
userDeilverNumber++;
userResume = userResumeQueue.Dequeue();
continue;
}
if (!positionQueue.Any())
{
return;
}
position = positionQueue.Dequeue();
userDeilverNumber = 0;
continue;
}
var deilverCount = userResume.Item2;
if (++deilverCount < singleUserDeilverCount)
{
userResumeQueue.Enqueue(new Tuple <dynamic, int>(userResume.Item1, deilverCount));
}
break;
}while (true);
#endregion
#region 变更任务状态及添加投递记录
using (var db = new MangningXssDBEntities())
{
var taskId = (int)position.TaskId;
var deilverTask = db.ZhaopinDeilverTask.FirstOrDefault(f => f.Id == taskId);
if (deilverTask != null)
{
deilverTask.ActualDeilverCount += 1;
if (deilverTask.ActualDeilverCount == deilverTask.ExpectedDeilverCount)
{
deilverTask.Status = 1;
deilverTask.CompleteTime = DateTime.UtcNow;
}
}
else
{
LogFactory.Warn("数据库未找到对应的任务！任务ID：" + taskId);
}
db.ZhaopinDeilveryRecord.Add(new ZhaopinDeilveryRecord
{
CompanyId = position.CompanyId,
PositionId = position.PositionId,
UserId = userResume.Item1.UserId,
CreateTime = DateTime.UtcNow
});
db.TransactionSaveChanges();
}
#endregion
LogFactory.Info($"投递成功！用户：{userResume.Item1.Username} 今日投递数：{userResume.Item2 + 1} 职位：{position.Name} 还剩 {positionQueue.Count} 个职位待投递");
if (userResumeQueue.Count != 0)
{
Thread.Sleep(1000 * 5 / userResumeQueue.Count);
}
}
}
/// <summary>
/// 下载投递简历列表
/// </summary>
/// <param name="path"></param>
/// <param name="cookieStr"></param>
public void DownOldSystemResumeDetail(string path, string cookieStr)
{
var count = 0;
while (true)
{
var requestResult = RequestFactory.QueryRequest("https://rd2.zhaopin.com/rdapply/resumes/apply/search?SF_1_1_38=7,9&orderBy=CreateTime", "PageList2=&DColumn_hidden=&searchKeyword=&curSubmitRecord=1586&curMaxPageNum=80&PageList2=1&buttonAsse=%E5%AF%BC%E5%85%A5%E6%B5%8B%E8%AF%84%E7%B3%BB%E7%BB%9F&buttonInfo=%E5%8F%91%E9%80%9A%E7%9F%A5%E4%BF%A1&SF_1_1_50=1&SF_1_1_51=-1&SF_1_1_45=&SF_1_1_44=&SF_1_1_52=0&SF_1_1_49=0&IsInvited=0&position_city=%5B%25%25POSITION_CITY%25%25%5D&deptName=&select_unique_id=&selectedResumeList=&PageNo=&PosState=&MinRowID=&MaxRowID=2722819791&RowsCount=123&PagesCount=5&PageType=0&CurrentPageNum=1&Position_IDs=%5B%25%25POSITION_IDS%25%25%5D&Position_ID=%5B%25%25POSITION_ID%25%25%5D&SortType=0&isCmpSum=0&SelectIndex_Opt=0&Resume_count=0&CID=56211453&forwardingEmailList=&click_search_op_type=-1&X-Requested-With=XMLHttpRequest", RequestEnum.POST, _cookieContainer);
if (!requestResult.IsSuccess)
{
this.AsyncSetLog(this.tbx_Log, "请求异常！异常原因：" + requestResult.ErrorMsg);
return;
}
var matches = Regex.Matches(requestResult.Data, "(?s)javascript:ViewOneResume.+?'(\\d+)'\\);\" href=\"(.+?)\".+?>(.+?)</a.+?<td title=\"(.+?)\"");
if (requestResult.Data.Contains("因请求量过大导致系统无法处理您的请求，您需要通过验证才能继续后续的操作！"))
{
this.AsyncSetLog(this.tbx_Log, "搜索简历列表失败！出现验证码！");
using (var db = new MangningXssDBEntities())
{
if (!db.ZhaopinCheckCode.Any(a => a.Account == companyName && (a.Status == 0 || a.Status == 1)))
{
db.ZhaopinCheckCode.Add(new ZhaopinCheckCode
{
Account = companyName,
Cookie = cookieStr,
CreateTime = DateTime.Now,
Status = 0,
Type = 1
});
db.SaveChanges();
}
}
Thread.Sleep(retryInterval);
continue;
}
if (matches.Count == 0)
{
Thread.Sleep(10 * 1000);
continue;
}
var ids = string.Empty;
foreach (Match match in matches)
{
try
{
var filePath = $"{path}\\{match.Result("$3").Replace("\\", "").Replace("/", "")}_{DateTime.Parse(match.Result("$4")):yyyy-MM-dd HH：mm：ss}.txt";
while (true)
{
requestResult = RequestFactory.QueryRequest($"https:{match.Result("$2")}", cookieContainer: _cookieContainer);
if (!requestResult.IsSuccess)
{
this.AsyncSetLog(this.tbx_Log, "请求异常！异常原因：" + requestResult.ErrorMsg);
return;
}
if (requestResult.Data.Contains("因请求量过大导致系统无法处理您的请求，您需要通过验证才能继续后续的操作！"))
{
this.AsyncSetLog(this.tbx_Log, "查看简历详情失败！出现验证码！");
using (var db = new MangningXssDBEntities())
{
if (!db.ZhaopinCheckCode.Any(a => a.Account == companyName && (a.Status == 0 || a.Status == 1)))
{
db.ZhaopinCheckCode.Add(new ZhaopinCheckCode
{
Account = companyName,
Cookie = cookieStr,
CreateTime = DateTime.Now,
Status = 0,
Type = 1
});
db.SaveChanges();
}
}
Thread.Sleep(retryInterval);
continue;
}
Thread.Sleep(interval);
ids += match.Result("$1") + "%3B";
break;
}
File.WriteAllText(filePath, requestResult.Data);
this.AsyncSetLog(this.tbx_Log, $"下载成功！第 {++count} 份 Path => {filePath}");
var htmlDocument = new HtmlDocument();
htmlDocument.LoadHtml(requestResult.Data);
resumeQueue.Enqueue(new KeyValuePair <HtmlDocument, DateTime>(htmlDocument, DateTime.Parse(match.Result("$4"))));
}
catch (Exception ex)
{
this.AsyncSetLog(this.tbx_Log, "程序异常！异常消息：" + ex.Message);
}
}
requestResult = RequestFactory.QueryRequest("https://rd2.zhaopin.com/RdApply/Resumes/Apply/SetResumeState", $"ids={ids}&oldResumeState=1&resumeState=4", RequestEnum.POST, _cookieContainer);
var jsonObj = JsonConvert.DeserializeObject <dynamic>(requestResult.Data);
if ((int)jsonObj.Code != 200)
{
this.AsyncSetLog(this.tbx_Log, $"标记简历为不合适失败！Message = {(string)jsonObj.Message}");
continue;
}
this.AsyncSetLog(this.tbx_Log, "标记简历为不合适成功！");
}
this.AsyncSetLog(this.tbx_Log, "简历处理完成！");
this.RunInMainthread(() =>
{
this.btn_StartDownload.Enabled = true;
});
}
/// <summary>
/// 检查用户账户是否被封禁
/// </summary>
/// <param name="userResume"></param>
/// <param name="code"></param>
/// <returns></returns>
private static bool CheckUserIsUseable(dynamic userResume, out int code)
{
code = 0;
var date = DateTime.Now.Date.AddHours(-8);
if (checkRecordDictionary.ContainsKey(userResume.UserId) && checkRecordDictionary[userResume.UserId] > date)
{
return(true);
}
while (true)
{
if (!localCompanyInfoQueue.Any())
{
using (var db = new MangningXssDBEntities())
{
var companyPositionList = (from a in db.ZhaoPinCompany
join b in db.ZhaopinPosition on a.Id equals b.CompanyId
join c in db.ZhaopinStaff on a.Id equals c.CompanyId
where a.Source.Contains("MANUAL") && !string.IsNullOrEmpty(c.Cookie) && b.IsEnable
select new
{
a.Name,
CompanyId = a.Id,
PositionNumber = b.Number,
c.Cookie
}).ToList();
if (companyPositionList.Any())
{
companyPositionList.ForEach(f =>
{
localCompanyInfoQueue.Enqueue(f);
});
}
else
{
//todo:调用通知API
LogFactory.Warn("无有效本地公司 Cookie 校验帐号是否可用！");
Thread.Sleep(TimeSpan.FromSeconds(30));
continue;
}
}
}
while (true)
{
if (!localCompanyInfoQueue.Any())
{
break;
}
var localCompanyInfo = localCompanyInfoQueue.Dequeue();
var cookieStr = (string)localCompanyInfo.Cookie;
var isLoginResult = RequestFactory.QueryRequest("https://ihr.zhaopin.com/home/assetcount.do", cookieContainer: cookieStr.Serialize("zhaopin.com"));
if (!isLoginResult.IsSuccess)
{
LogFactory.Warn($"校验公司 Cookie 是否有效请求异常！公司：{localCompanyInfo.Name}，异常信息：{isLoginResult.ErrorMsg}");
continue;
}
var obj = JsonConvert.DeserializeObject(isLoginResult.Data) as JObject;
if ((string)obj?["code"] != "200")
{
var id = (int)localCompanyInfo.CompanyId;
using (var db = new MangningXssDBEntities())
{
var companyStaff = db.ZhaopinStaff.FirstOrDefault(f => f.CompanyId == id);
if (companyStaff != null)
{
companyStaff.Cookie = "";
}
db.TransactionSaveChanges();
}
var count = localCompanyInfoQueue.Count;
for (var i = 0; i < count; i++)
{
var temp = localCompanyInfoQueue.Dequeue();
if (temp.CompanyId != localCompanyInfo.CompanyId)
{
localCompanyInfoQueue.Enqueue(temp);
}
}
LogFactory.Warn($"公司 Cookie 失效！公司：{localCompanyInfo.Name}，异常信息：{obj?["message"]}");
continue;
}
var userCookieStr = (string)userResume.Cookie;
var deilverResult = RequestFactory.QueryRequest($"https://my.zhaopin.com/v5/FastApply/resumeinfo.aspx?t=3&j={localCompanyInfo.PositionNumber}&j2=&so=&su=&ff=ssb&rv={userResume.ResumeNumber}_1&rl=1&cl=&sd=0&fd=&c=jsonp7u3t5v&_=1493174565322", cookieContainer: userCookieStr.Serialize("zhaopin.com"));
if (!deilverResult.IsSuccess)
{
LogFactory.Warn($"投递简历请求异常！简历编号：{userResume.ResumeNumber}，职位编号：{localCompanyInfo.PositionNumber}，异常信息：{deilverResult.ErrorMsg}");
continue;
}
var message = JsonConvert.DeserializeObject <dynamic>(Regex.Match(deilverResult.Data, @"(\{.+?\})").Result("$1"));
var status = message.postBackInfo.ToString().Split('_');
if (message.loginstatus.ToString() != "6")
{
return(true);
}
if (message.loginstatus.ToString() == "7_Position has Down")
{
LogFactory.Warn($"该职位已下架！职位编号：{localCompanyInfo.PositionNumber}");
using (var db = new MangningXssDBEntities())
{
var positionNumber = (string)localCompanyInfo.PositionNumber;
var positionDefault = db.ZhaopinPosition.FirstOrDefault(f => f.Number == positionNumber);
if (positionDefault != null)
{
positionDefault.IsEnable = false;
}
else
{
LogFactory.Warn("数据库未找到对应的职位！职位 Number：" + positionNumber);
}
db.TransactionSaveChanges();
}
continue;
}
if (status[3].ToString() == "0")
{
Console.WriteLine($"投递本地公司失败！，职位 7 天内投递过！公司：{localCompanyInfo.Name} 职位编号：{localCompanyInfo.PositionNumber} 用户：{userResume.Username}");
if (!localCompanyInfoQueue.Any())
{
LogFactory.Warn($"本地可用公司的所有职位均在7天内投递过！用户：{userResume.Username}");
}
continue;
}
Console.WriteLine($"投递本地公司成功！公司：{localCompanyInfo.Name}，用户：{userResume.Username}");
Thread.Sleep(TimeSpan.FromSeconds(5));
var token = cookieStr.Substring(cookieStr.IndexOf("Token=", StringComparison.Ordinal) + 6, 32);
var dataResult = RequestFactory.QueryRequest($"https://ihr.zhaopin.com/resumemanage/resumelistbykey.do?access_token={token}&v=0.47046618467879875", "startNum=0&rowsCount=30&ageStart=&ageEnd=&workYears=&sex=&edu=&liveCity=&hopeWorkCity=&upDate=&companyName=&exclude=&keywords=&onlyLastWork=false&orderFlag=deal&countFlag=1&jobNo=&pageType=all&source=1%3B2%3B5&sort=time", RequestEnum.POST, cookieStr.Serialize("zhaopin.com"));
if (!dataResult.IsSuccess)
{
LogFactory.Warn($"搜索简历列表请求异常！公司ID：{localCompanyInfo.CompanyId}，异常信息：{deilverResult.ErrorMsg}");
continue;
}
var jsonObj = JsonConvert.DeserializeObject(dataResult.Data) as JObject;
if ((string)jsonObj?["code"] != "1")
{
var id = (int)localCompanyInfo.CompanyId;
using (var db = new MangningXssDBEntities())
{
var companyStaff = db.ZhaopinStaff.FirstOrDefault(f => f.CompanyId == id);
if (companyStaff != null)
{
companyStaff.Cookie = "";
}
db.TransactionSaveChanges();
}
var count = localCompanyInfoQueue.Count;
for (var i = 0; i < count; i++)
{
var temp = localCompanyInfoQueue.Dequeue();
if (temp.CompanyId != localCompanyInfo.CompanyId)
{
localCompanyInfoQueue.Enqueue(temp);
}
}
LogFactory.Warn($"企业帐号简历列表获取异常！公司ID：{localCompanyInfo.CompanyId}，异常信息：{jsonObj?["message"]}");
continue;
}
var resumes = jsonObj?["data"]["deal"]["results"] as JArray;
if (resumes == null)
{
LogFactory.Warn($"搜索简历列表 Josn 解析异常！公司ID：{localCompanyInfo.CompanyId}，异常 Josn：{dataResult.Data}");
continue;
}
localCompanyInfoQueue.Enqueue(localCompanyInfo);
var resume = resumes.FirstOrDefault(a => a["userId"] == userResume.UserId && Convert.ToDateTime(a["createTime"]) > DateTime.UtcNow.AddMinutes(-5));
if (resume != null)
{
if (!resume.ToString().Contains("<script"))
{
LogFactory.Warn($"简历未处理，用户：{userResume.Username}");
code = -1;
return(false);
}
checkRecordDictionary[userResume.UserId] = DateTime.UtcNow;
Console.WriteLine($"用户可用！公司：{localCompanyInfo.Name}，用户：{userResume.Username}");
return(true);
}
Console.WriteLine($"用户不可用！公司：{localCompanyInfo.Name}，用户：{userResume.Username}");
return(false);
}
}
}
/// <summary>
/// 组合简历
/// </summary>
private static void PortfolioResume()
{
while (true)
{
try
{
ZhaopinResume resume;
var stopwatch = new Stopwatch();
stopwatch.Restart();
if (!resumeQueue.TryDequeue(out resume))
{
Thread.Sleep(100);
continue;
}
var filePath = $"{uploadFilePath}{resume.Id}.json";
if (File.Exists(filePath))
{
uploadQueue.Enqueue(filePath);
continue;
}
string userId;
string cellphone;
string email;
using (var db = new MangningXssDBEntities())
{
var user = db.ZhaopinUser.AsNoTracking().FirstOrDefault(f => f.Id == resume.UserId);
if (user == null)
{
continue;
}
userId = user.Id.ToString();
cellphone = user.Cellphone;
email = user.Email;
using (var stream = new MemoryStream())
{
if (mangningOssClient.DoesObjectExist(mangningBucketName, $"Zhaopin/Resume/{resume.Id}"))
{
var bytes = new byte[1024];
int len;
var streamContent = mangningOssClient.GetObject(mangningBucketName, $"Zhaopin/Resume/{resume.Id}").Content;
while ((len = streamContent.Read(bytes, 0, bytes.Length)) > 0)
{
stream.Write(bytes, 0, len);
}
var resumeContent = Encoding.UTF8.GetString(GZip.Decompress(stream.ToArray()));
var resumeObj = JsonConvert.DeserializeObject <dynamic>(resumeContent);
var resumeDetail = JsonConvert.DeserializeObject(resumeObj.detialJSonStr.ToString());
resumeDetail.DateModified = user.ModifyTime.ToLocalTime();
resumeDetail.DateCreated = user.CreateTime?.ToLocalTime() ?? resumeDetail.DateCreated;
resumeDetail.DateLastReleased = resume.RefreshTime.Value.ToLocalTime();
resumeDetail.DateLastViewed = resume.RefreshTime.Value.ToLocalTime();
resumeObj.detialJSonStr = resumeDetail;
resumeContent = JsonConvert.SerializeObject(resumeObj);
File.WriteAllText(filePath, resumeContent);
uploadQueue.Enqueue(filePath);
continue;
}
if (mangningOssClient.DoesObjectExist(mangningBucketName, $"WatchResume/{resume.Id}"))
{
var bytes = new byte[1024];
int len;
var streamContent = mangningOssClient.GetObject(mangningBucketName, $"WatchResume/{resume.Id}").Content;
while ((len = streamContent.Read(bytes, 0, bytes.Length)) > 0)
{
stream.Write(bytes, 0, len);
}
var resumeContent = Encoding.UTF8.GetString(GZip.Decompress(stream.ToArray()));
var resumeObj = JsonConvert.DeserializeObject <dynamic>(resumeContent);
resumeObj.userDetials.mobilePhone = user.Cellphone;
resumeObj.userDetials.email = user.Email;
var resumeDetail = JsonConvert.DeserializeObject(resumeObj.detialJSonStr.ToString());
resumeDetail.DateModified = user.ModifyTime.ToLocalTime();
resumeDetail.DateCreated = user.CreateTime.Value.ToLocalTime();
resumeDetail.DateLastReleased = resume.RefreshTime.Value.ToLocalTime();
resumeDetail.DateLastViewed = resume.RefreshTime.Value.ToLocalTime();
resumeObj.detialJSonStr = resumeDetail;
resumeContent = JsonConvert.SerializeObject(resumeObj);
using (var jsonStream = new MemoryStream(GZip.Compress(Encoding.UTF8.GetBytes(resumeContent))))
{
mangningOssClient.PutObject(mangningBucketName, $"Zhaopin/Resume/{resume.Id}", jsonStream);
}
File.WriteAllText(filePath, resumeContent);
uploadQueue.Enqueue(filePath);
continue;
}
var zhaopinResume = db.ZhaopinResume.FirstOrDefault(f => f.Id == resume.Id);
if (zhaopinResume != null)
{
zhaopinResume.Flag = 0x2;
}
}
db.SaveChanges();
stopwatch.Stop();
}
using (var bdcDb = new BadoucaiAliyunDBEntities())
{
bdcDb.CoreResumeZhaopin.Add(new CoreResumeZhaopin
{
Cellphone = cellphone,
Email = email,
ResumeKey = userId,
Type = "ResumeUserId",
IsMatched = false
});
bdcDb.SaveChanges();
}
Console.WriteLine($"{DateTime.Now} > 简历未找到 Josn 源！ResumeId = {resume.Id}, UserId = {userId}, Elapsed = {stopwatch.ElapsedMilliseconds} ms.");
}
catch (Exception ex)
{
Trace.TraceError(ex.ToString());
}
}
}
/// <summary>
/// 按条件搜索简历
/// </summary>
/// <param name="cookieContainer"></param>
private static void SearchResumes(CookieContainer cookieContainer)
{
while (true)
{
ZhaopinSearchCondition condition;
if (count >= refreshCount)
{
break;
}
if (!conditionQueue.TryDequeue(out condition))
{
Thread.Sleep(100);
break;
}
var pageIndex = 1;
var pageTotal = 1;
var total = 0;
while (pageIndex < pageTotal + 1)
{
if (count >= refreshCount)
{
break;
}
// 2041：宝安 635：南京 653：杭州 639：苏州 765：深圳
var param = string.IsNullOrEmpty(keyword) ? string.Empty : $"SF_1_1_1={keyword}&";
param += string.IsNullOrEmpty(position) ? string.Empty : $"SF_1_1_2={position}&";
param += string.IsNullOrEmpty(industry) ? string.Empty : $"SF_1_1_3={industry}&";
var url = $"https://ihrsearch.zhaopin.com/Home/ResultForCustom?{param}SF_1_1_6={city}&SF_1_1_8={condition.Age}%2C{condition.Age}&SF_1_1_9={condition.Gender}&SF_1_1_5={condition.Degrees}%2C{condition.Degrees}&orderBy=BIRTH_YEAR%2C0&SF_1_1_27=0&SF_1_1_7={date}%2C9&exclude=1&pageIndex={pageIndex}&pageSize=60";
var requestResult = RequestFactory.QueryRequest(url, cookieContainer: cookieContainer, referer: url);
if (!requestResult.IsSuccess)
{
Trace.TraceError($"{DateTime.Now} > Search Error ! Message = {requestResult.ErrorMsg} Condition = {JsonConvert.SerializeObject(condition)}.");
continue;
}
if (pageTotal == 1)
{
var match = Regex.Match(requestResult.Data, "(?s)<span>(\\d+)</span>份简历.+?rd-resumelist-pageNum\">1/(\\d+)</span>");
if (!match.Success)
{
if (requestResult.Data.Contains("text/javascript\" r='m'"))
{
Trace.TraceWarning($"{DateTime.Now} > Cookie Expired !");
}
else
{
Trace.TraceWarning($"{DateTime.Now} > Condition search error ! Page content = {requestResult.Data}");
}
return;
}
total = Convert.ToInt32(match.Result("$1"));
pageTotal = Convert.ToInt32(match.Result("$2"));
totalCount += total;
if (total == 0)
{
Console.WriteLine($"{DateTime.Now} > Search results are empty !");
break;
}
}
Console.WriteLine($"{DateTime.Now} > 第 {pageIndex} 页 共 {pageTotal} 页 {total} 个结果");
pageIndex++;
var matchs = Regex.Matches(requestResult.Data, "(?s)RedirectToRd/([^\r\n]+?)','([^\r\n]+?)','([^\r\n]+?)',this\\);this.+?(\\d{2}-\\d{2}-\\d{2})");
foreach (Match item in matchs)
{
try
{
if (count >= refreshCount)
{
break;
}
var number = item.Result("$1").Substring(0, 10);
DateTime updateDateTime;
if (!DateTime.TryParse(item.Result("$4"), out updateDateTime))
{
continue;
}
using (var db = new MangningXssDBEntities())
{
var resume = db.ZhaopinResume.FirstOrDefault(f => f.RandomNumber == number);
if (resume != null)
{
if (resume.Flag == 15)
{
#region 刷新简历更新时间
//if (resume.RefreshTime != null && updateDateTime.Date == resume.RefreshTime.Value.Date) continue;
resume.RefreshTime = updateDateTime;
resume.Flag = 14;
var filePath = $@"D:\Badoucai\Resume\LocationJson\{resume.Id}.json";
using (var stream = new MemoryStream())
{
if (!mangningOssClient.DoesObjectExist(mangningBucketName, $"Zhaopin/Resume/{resume.Id}"))
{
continue;
}
var bytes = new byte[1024];
int len;
var streamContent = mangningOssClient.GetObject(mangningBucketName, $"Zhaopin/Resume/{resume.Id}").Content;
while ((len = streamContent.Read(bytes, 0, bytes.Length)) > 0)
{
stream.Write(bytes, 0, len);
}
var resumeContent = Encoding.UTF8.GetString(GZip.Decompress(stream.ToArray()));
var jsonObj = JsonConvert.DeserializeObject <dynamic>(resumeContent);
dynamic detialJSonStr;
try
{
detialJSonStr = jsonObj.detialJSonStr;
if (!string.IsNullOrEmpty((string)jsonObj.detialJSonStr.DateModified))
{
jsonObj.detialJSonStr = JsonConvert.SerializeObject(jsonObj.detialJSonStr);
}
}
catch (Exception)
{
detialJSonStr = JsonConvert.DeserializeObject <dynamic>((string)jsonObj.detialJSonStr);
}
detialJSonStr.DateLastReleased = updateDateTime;
detialJSonStr.DateModified = updateDateTime;
jsonObj.detialJSonStr = detialJSonStr;
var newResumeContent = JsonConvert.SerializeObject(jsonObj);
using (var jsonStream = new MemoryStream(GZip.Compress(Encoding.UTF8.GetBytes(newResumeContent))))
{
mangningOssClient.PutObject(mangningBucketName, $"Zhaopin/Resume/{resume.Id}", jsonStream);
}
File.WriteAllText(filePath, newResumeContent);
}
#endregion
}
if (resume.Flag == 2)
{
var user = db.ZhaopinUser.FirstOrDefault(f => f.Id == resume.UserId);
if (user == null)
{
continue;
}
WatchResumeDetail(item, cookieContainer, user.Cellphone, user.Email);
}
}
else
{
var incompleteResume = db.ZhaopinIncompleteResume.FirstOrDefault(f => f.ResumeNumber == number);
if (incompleteResume == null)
{
continue;
}
if (WatchResumeDetail(item, cookieContainer, incompleteResume.Cellphone, incompleteResume.Email))
{
incompleteResume.CompletionTime = DateTime.Now;
}
}
db.SaveChanges();
Interlocked.Increment(ref count);
if (resume != null)
{
sb.AppendLine(resume.Id.ToString());
}
}
}
catch (Exception ex)
{
Trace.TraceError(ex.ToString());
}
}
Console.WriteLine($"{DateTime.Now} > Total = {totalCount}, Update Count = {count}, Cookie Count = {cookieQueue.Count}, Condition Count = {conditionQueue.Count}. ");
}
}
}
public void Watch()
{
var cookieQueue = new ConcurrentQueue <KeyValuePair <KeyValuePair <int, string>, CookieContainer> >();
using (var db = new MangningXssDBEntities())
{
var companyArr = db.ZhaoPinCompany.Where(w => w.Source.Contains("MANUAL")).Select(s => s.Id).ToArray();
var paramArr = db.ZhaopinStaff.Where(w => companyArr.Any(a => a == w.CompanyId) && !string.IsNullOrEmpty(w.Cookie)).Select(s => new { s.CompanyId, s.Username, s.Cookie }).ToArray();
foreach (var item in paramArr)
{
cookieQueue.Enqueue(new KeyValuePair <KeyValuePair <int, string>, CookieContainer>(new KeyValuePair <int, string>(item.CompanyId, item.Username), item.Cookie.Serialize(".zhaopin.com")));
}
}
using (var db = new MangningXssDBEntities())
{
var list = db.ZhaopinCleaningProcedure.Where(w => w.IsEnable && !string.IsNullOrEmpty(w.Cookie));
foreach (var item in list)
{
cookieQueue.Enqueue(new KeyValuePair <KeyValuePair <int, string>, CookieContainer>(new KeyValuePair <int, string>(0, item.Account), item.Cookie.Serialize(".zhaopin.com")));
}
}
#region 插入查询条件
for (var degrees = 5; degrees < 16; degrees += 2)
{
for (var age = 20; age <= 28; age++)
{
queue.Enqueue(new
{
Degrees = degrees,
Age = age
});
}
}
#endregion
Console.WriteLine($"已获取 Cookie 数 => {cookieQueue.Count}");
for (var j = 0; j < 4; j++)
{
Task.Run(() =>
{
try
{
while (true)
{
KeyValuePair <KeyValuePair <int, string>, CookieContainer> temp;
if (!cookieQueue.TryDequeue(out temp))
{
Thread.Sleep(1000);
continue;
}
while (true)
{
dynamic condition;
if (!queue.TryDequeue(out condition))
{
Console.WriteLine($"Company => {temp.Key.Value} 找不到可搜索的条件！");
Thread.Sleep(1000);
continue;
}
var pageIndex = 1;
var pageTotal = 1;
var total = 0;
var isbreak = false;
while (pageIndex < pageTotal + 1)
{
var url = $"https://ihrsearch.zhaopin.com/Home/ResultForCustom?SF_1_1_7=1%2C9&SF_1_1_5={condition.Degrees}%2C{condition.Degrees}&SF_1_1_18=530%3B854%3B531&SF_1_1_8={condition.Age}%2C{condition.Age}&SF_1_1_27=0&orderBy=DATE_MODIFIED,1&pageSize=60&exclude=1";
var requestResult = RequestFactory.QueryRequest(url, cookieContainer: temp.Value, referer: url);
if (!requestResult.IsSuccess)
{
LogFactory.Warn($"Company => {temp.Key.Value} 条件搜索异常！异常原因=>{requestResult.ErrorMsg} Condition=>{JsonConvert.SerializeObject(condition)}");
continue;
}
if (pageTotal == 1)
{
var match = Regex.Match(requestResult.Data, "(?s)<span>(\\d+)</span>份简历.+?rd-resumelist-pageNum\">1/(\\d+)</span>");
if (!match.Success)
{
if (requestResult.Data.Contains("text/javascript\" r='m'"))
{
LogFactory.Warn($"Cookie 过期，过期用户 => {temp.Key.Value}");
}
else
{
LogFactory.Warn($"Company => {temp.Key.Value} 条件搜索异常！返回页面解析异常！{requestResult.Data}");
}
isbreak = true;
break;
}
total = Convert.ToInt32(match.Result("$1"));
pageTotal = Convert.ToInt32(match.Result("$2"));
if (total == 0)
{
Console.WriteLine("该条件无搜索结果！");
break;
}
}
Console.WriteLine($"Company => {temp.Key.Value} 第 {pageIndex} 页 共 {pageTotal} 页 {total} 个结果");
pageIndex++;
var matchs = Regex.Matches(requestResult.Data, "(?s)RedirectToRd/([^\r\n]+?)','([^\r\n]+?)','([^\r\n]+?)',this\\);this.+?(\\d{2}-\\d{2}-\\d{2})");
using (var db = new MangningXssDBEntities())
{
foreach (Match item in matchs)
{
var number = item.Result("$1").Substring(0, 10);
var numberParam = item.Result("$1").Substring(0, item.Result("$1").IndexOf("?", StringComparison.Ordinal));
if (!hashSet.Add(number))
{
continue;
}
db.ZhaopinResumeNumber.Add(new ZhaopinResumeNumber {
ResumeNumber = number
});
var resume = db.ZhaopinResume.FirstOrDefault(f => f.RandomNumber == number);
if (resume == null)
{
continue;
}
var user = db.ZhaopinUser.FirstOrDefault(f => f.Id == resume.UserId && !string.IsNullOrEmpty(f.Cellphone));
if (user == null)
{
continue;
}
requestResult = RequestFactory.QueryRequest($"http://ihr.zhaopin.com/resumesearch/getresumedetial.do?resumeNo={numberParam}&searchresume=1&resumeSource=1&keyword=%E8%AF%B7%E8%BE%93%E5%85%A5%E7%AE%80%E5%8E%86%E5%85%B3%E9%94%AE%E8%AF%8D%EF%BC%8C%E5%A4%9A%E5%85%B3%E9%94%AE%E8%AF%8D%E5%8F%AF%E7%94%A8%E7%A9%BA%E6%A0%BC%E5%88%86%E9%9A%94&t={item.Result("$2")}&k={item.Result("$3")}&v=undefined&version=3&openFrom=1", cookieContainer: temp.Value);
if (!requestResult.IsSuccess)
{
LogFactory.Warn($"Company => {temp.Key.Value} 简历详情查看异常！异常原因=>{requestResult.ErrorMsg} ResumeNumber=>{number}");
continue;
}
try
{
var jsonObj = JsonConvert.DeserializeObject <dynamic>(requestResult.Data);
if ((int)jsonObj["code"] != 1)
{
LogFactory.Warn($"Company => {temp.Key.Value} ResumeNumber => {number} 查看详情异常 信息：{(string)jsonObj["message"]}");
if (((string)jsonObj["message"]).Contains("当日查看简历已达上限"))
{
isbreak = true;
break;
}
continue;
}
var resumeData = jsonObj.data;
resumeData.userDetials.mobilePhone = user.Cellphone;
resumeData.userDetials.email = user.Email;
File.WriteAllText($@"D:\ResumeBusiness\{number}.json", JsonConvert.SerializeObject(resumeData));
Console.WriteLine($"Company => {temp.Key.Value} 查看简历成功！ResumeNumber => {number}");
}
catch (Exception ex)
{
while (true)
{
if (ex.InnerException == null)
{
break;
}
ex = ex.InnerException;
}
LogFactory.Error($"保存简历信息异常！异常信息：{ex.Message} 响应信息：{requestResult.Data}");
}
}
db.SaveChanges();
}
if (isbreak)
{
break;
}
}
if (isbreak)
{
break;
}
}
}
}
catch (Exception ex)
{
while (true)
{
if (ex.InnerException == null)
{
break;
}
ex = ex.InnerException;
}
LogFactory.Error($"Watch异常！异常信息：{ex.Message} {ex.StackTrace}");
}
});
}
}
/// <summary>
/// 标记简历
/// </summary>
/// <param name="jsonContent"></param>
/// <param name="resumeId"></param>
private static void FlagResume(string jsonContent, int resumeId)
{
using (var db = new MangningXssDBEntities())
{
var resume = db.ZhaopinResume.FirstOrDefault(f => f.Id == resumeId);
if (resume?.Flag == 0xF)
{
badoucaiOssClient.DeleteObject(badoucaiBucketName, $"Zhaopin/{resumeId}");
resume.IncludeTime = DateTime.UtcNow;
db.SaveChanges();
return;
}
var resumeIdStr = resumeId.ToString();
if (jsonContent.Contains("detialJSonStr"))
{
var jsonObj = JsonConvert.DeserializeObject <dynamic>(jsonContent);
//var userId = (int)jsonObj.userDetials.userMasterId;
if (string.IsNullOrWhiteSpace((string)jsonObj.userDetials.mobilePhone))
{
var user = resume != null?db.ZhaopinUser.FirstOrDefault(f => f.Id == resume.UserId && !string.IsNullOrEmpty(f.Cellphone)) : null;
if (user == null)
{
using (var adb = new BadoucaiAliyunDBEntities())
{
var reference = adb.CoreResumeReference.AsNoTracking().FirstOrDefault(f => f.Id == resumeIdStr);
CoreResumeSummary summary = null;
if (reference != null)
{
summary = adb.CoreResumeSummary.AsNoTracking().FirstOrDefault(f => f.Id == reference.ResumeId);
}
if (summary != null)
{
jsonObj.userDetials.mobilePhone = summary.Cellphone;
jsonObj.userDetials.email = summary.Email;
}
}
}
else
{
jsonObj.userDetials.mobilePhone = user.Cellphone;
jsonObj.userDetials.email = user.Email;
}
}
File.WriteAllText($"{jsonFilePath}{resumeId}", jsonContent);
#region 被注释的代码
//var flag = string.IsNullOrEmpty((string)jsonObj.userDetials.mobilePhone) ? (short)0x0 : (short)0xF;
//dynamic detialJSonStr;
//try
//{
// detialJSonStr = jsonObj.detialJSonStr;
// if (!string.IsNullOrEmpty((string)jsonObj.detialJSonStr.DateModified))
// {
// jsonObj.detialJSonStr = JsonConvert.SerializeObject(jsonObj.detialJSonStr);
// }
//}
//catch (Exception)
//{
// detialJSonStr = JsonConvert.DeserializeObject<dynamic>((string)jsonObj.detialJSonStr);
//}
//if (resume == null)
//{
// db.ZhaopinResume.Add(new ZhaopinResume
// {
// Id = resumeId,
// RandomNumber = ((string)jsonObj.resumeNo).Substring(0, 10),
// UserId = userId,
// RefreshTime = BaseFanctory.GetTime((string)detialJSonStr.DateModified).ToUniversalTime(),
// UpdateTime = DateTime.UtcNow,
// UserExtId = (string)detialJSonStr.UserMasterExtId,
// Source = "XSS",
// Flag = flag
// });
// handlerFlag = "Insert";
//}
//else
//{
// resume.RandomNumber = ((string)jsonObj.resumeNo).Substring(0, 10);
// resume.UserId = userId;
// resume.RefreshTime = BaseFanctory.GetTime((string)detialJSonStr.DateModified).ToUniversalTime();
// resume.UpdateTime = DateTime.UtcNow;
// resume.Flag = flag;
// handlerFlag = "Update";
//}
//db.ZhaopinUser.AddOrUpdate(new ZhaopinUser
//{
// Id = userId,
// Cellphone = (string)jsonObj.userDetials.mobilePhone,
// CreateTime = BaseFanctory.GetTime((string)detialJSonStr.DateCreated).ToUniversalTime(),
// Email = (string)jsonObj.userDetials.email,
// ModifyTime = BaseFanctory.GetTime((string)detialJSonStr.DateModified).ToUniversalTime(),
// Name = (string)jsonObj.userDetials.userName,
// Source = "XSS",
// UpdateTime = DateTime.UtcNow
//});
//var jsonResume = JsonConvert.SerializeObject(jsonObj);
//if (flag == 0x0)
//{
// var path = $@"F:\ZhaopinOss\Resume\NoInformation\{resumeIdStr.Substring(0, 2)}\{resumeIdStr.Substring(2, 2)}";
// if (!Directory.Exists(path)) Directory.CreateDirectory(path);
// File.WriteAllText($@"{path}\{resumeIdStr}",jsonResume);
//}
//else
//{
// var path = $@"F:\ZhaopinOss\Resume\HaveInformation\{resumeIdStr.Substring(0, 2)}\{resumeIdStr.Substring(2, 2)}";
// if (!Directory.Exists(path)) Directory.CreateDirectory(path);
// File.WriteAllText($@"{path}\{resumeIdStr}", jsonResume);
// using (var stream = new MemoryStream(GZip.Compress(Encoding.UTF8.GetBytes(jsonResume))))
// {
// mangningClient.PutObject(mangningBucketName, $"Zhaopin/Resume/{resumeIdStr}", stream);
// }
//}
#endregion
badoucaiOssClient.DeleteObject(badoucaiBucketName, $"Zhaopin/{resumeId}");
}
else
{
if (jsonContent.StartsWith("\"<!DOCTYPE HTML>", StringComparison.OrdinalIgnoreCase) || jsonContent.StartsWith("<html", StringComparison.OrdinalIgnoreCase))
{
if (jsonContent.StartsWith("\"<!DOCTYPE HTML>", StringComparison.OrdinalIgnoreCase))
{
jsonContent = jsonContent.Substring(1);
jsonContent = jsonContent.Substring(0, jsonContent.Length - 1);
jsonContent = Regex.Unescape(jsonContent);
}
var updateTime = Regex.Match(jsonContent, "(?s)resumeUpdateTime\">(.+?)<.+?userName.+?alt=\"(.+?)\"").ResultOrDefault("$1", "");
var name = Regex.Match(jsonContent, "(?s)resumeUpdateTime\">(.+?)<.+?userName.+?alt=\"(.+?)\"").ResultOrDefault("$2", "").Replace("\\", "").Replace("/", "");
string fileName;
if (string.IsNullOrEmpty(updateTime) || string.IsNullOrEmpty(name))
{
fileName = $"{resumeId}";
}
else
{
fileName = $"{name}_{updateTime.Replace("年", "-").Replace("月", "-").Replace("日", "")}.txt";
}
File.WriteAllText($"{domFilePath}{fileName}", jsonContent);
}
else
{
Trace.WriteLine($"{DateTime.Now} > 简历格式异常！异常简历ID = {resumeId}, Content = {jsonContent}");
}
badoucaiOssClient.DeleteObject(badoucaiBucketName, $"Zhaopin/{resumeId}");
}
}
}
public void SaveResumeInfo(ResumeMatchResult model)
{
using (var xdb = new MangningXssDBEntities())
{
var user = xdb.ZhaopinUser.FirstOrDefault(f => f.Id == model.UserId);
if (user == null)
{
xdb.ZhaopinUser.Add(new ZhaopinUser
{
Id = model.UserId,
Source = "Watch",
ModifyTime = model.ModifyTime,
Cellphone = model.Cellphone,
Email = model.Email,
Name = model.Name,
UpdateTime = DateTime.UtcNow,
CreateTime = model.CreateTime
});
}
else
{
user.ModifyTime = model.ModifyTime;
user.UpdateTime = DateTime.UtcNow;
user.Name = model.Name;
}
var resumeEntity = xdb.ZhaopinResume.FirstOrDefault(f => f.Id == model.ResumeId);
var userExtId = Regex.IsMatch(model.UserExtId, @"^J[MRSL]\d{9}$") ? model.UserExtId : string.Empty;
if (resumeEntity == null)
{
xdb.ZhaopinResume.Add(new ZhaopinResume
{
Id = model.ResumeId,
RandomNumber = model.ResumeNumber,
UserId = model.UserId,
RefreshTime = model.ModifyTime,
UpdateTime = DateTime.UtcNow,
UserExtId = userExtId,
Source = "Watch"
});
}
else
{
resumeEntity.RandomNumber = model.ResumeNumber;
resumeEntity.UserId = model.UserId;
resumeEntity.RefreshTime = model.ModifyTime;
if (string.IsNullOrEmpty(resumeEntity.UserExtId))
{
resumeEntity.UserExtId = userExtId;
}
}
var cache = xdb.ZhaopinMatchedCache.FirstOrDefault(f => f.ResumeId == model.ResumeId);
if (cache == null)
{
xdb.ZhaopinMatchedCache.Add(new ZhaopinMatchedCache
{
ResumeId = model.ResumeId,
ModifyTime = model.ModifyTime,
Cellphone = model.Cellphone,
Email = model.Email,
Name = model.Name,
Path = model.Path,
ResumeNumber = model.ResumeNumber,
UserExtId = model.UserExtId,
UserId = model.UserId
});
}
else
{
cache.Cellphone = model.Cellphone;
cache.Email = model.Email;
cache.UserExtId = model.UserExtId;
}
var watched = xdb.ZhaopinWatchedResume.FirstOrDefault(f => f.Id == model.ResumeId);
if (watched == null)
{
xdb.ZhaopinWatchedResume.Add(new ZhaopinWatchedResume
{
Id = model.ResumeId,
ResumeNumber = model.ResumeNumber,
CompanyId = model.CompanyId,
WatchTime = DateTime.UtcNow
});
}
xdb.SaveChanges();
}
}
public void MatchZhaopin()
{
var resumeQueue = new ConcurrentQueue <OldResumeSummary>();
var cookieQueue = new ConcurrentQueue <QueueParam>();
var todayString = DateTime.Today.ToString("yyyy-MM-dd");
using (var db = new MangningXssDBEntities())
{
var companyArr = db.ZhaoPinCompany.Where(w => w.Source.Contains("MANUAL")).Select(s => s.Id).ToArray();
var paramArr = db.ZhaopinStaff.Where(w => companyArr.Any(a => a == w.CompanyId) && !string.IsNullOrEmpty(w.Cookie)).Select(s => new { s.CompanyId, s.Cookie }).ToArray();
foreach (var param in paramArr)
{
var task = db.ZhaopinResumeMatchLimit.FirstOrDefault(f => f.CompanyId == param.CompanyId);
if (task == null)
{
continue;
}
var todayTask = db.ZhaopinResumeMatchStatistic.FirstOrDefault(f => f.CompanyId == param.CompanyId && f.Date == todayString);
if (todayTask == null)
{
todayTask = new ZhaopinResumeMatchStatistic
{
Date = todayString,
CompanyId = param.CompanyId,
MatchedCount = 0,
SearchCount = 0,
WatchCount = 0
};
db.ZhaopinResumeMatchStatistic.Add(todayTask);
db.SaveChanges();
}
if (todayTask.SearchCount == task.DailySearchCount || todayTask.WatchCount == task.DailyWatchCount)
{
continue;
}
cookieQueue.Enqueue(new QueueParam
{
CompanyId = param.CompanyId,
Cookie = param.Cookie,
DailySeachCount = task.DailySearchCount,
DailyWatchCount = task.DailyWatchCount,
MatchCount = todayTask.MatchedCount,
SeachCount = todayTask.SearchCount,
WatchCount = todayTask.WatchCount + todayTask.MatchedCount
});
}
}
Task.Run(() =>
{
while (true)
{
if (resumeQueue.Count < 10)
{
using (var db = new ResumeMatchDBEntities())
{
var resumeList = db.OldResumeSummary.Where(w => w.ResumeId.Length == 10 && !w.IsMatched && w.Status == 0).Take(1000).ToList();
resumeList.ForEach(f =>
{
resumeQueue.Enqueue(f);
});
}
}
Thread.Sleep(TimeSpan.FromSeconds(1));
}
});
for (var j = 0; j < 16; j++)
{
Task.Run(() =>
{
using (var db = new ResumeMatchDBEntities())
{
while (true)
{
#region 匹配
OldResumeSummary resume;
if (!resumeQueue.TryDequeue(out resume))
{
continue;
}
var startNum = 0;
var resumeTemp = resume;
resume = db.OldResumeSummary.FirstOrDefault(f => f.Id == resumeTemp.Id);
QueueParam cookie;
if (!cookieQueue.TryDequeue(out cookie))
{
return;
}
try
{
using (var xdb = new MangningXssDBEntities())
{
var filePath = $@"E:\Data\智联招聘\{resume.Template}\{resume.ResumeId}.{Path.GetFileNameWithoutExtension(resume.Template)}";
if (!File.Exists(filePath))
{
LogFactory.Warn($"指定路径不存在！ResumeNumber=>{resume.ResumeId} Path=>{filePath}");
resume.Status = 1;
resume.MatchTime = DateTime.Now;
db.SaveChanges();
continue;
}
var sourceCode = File.ReadAllText(filePath);
//var sourceCode = string.Empty;
var genderMatch = Regex.Match(sourceCode, "(男|女)");
var gender = string.Empty;
if (genderMatch.Success)
{
gender = genderMatch.Value == "男" ? "1" : "2";
}
var matchs = Regex.Matches(sourceCode, "[\u4e00-\u9fa5]{4,11}有限公司[\u4e00-\u9fa5]{0,6}");
var companys = new List <string>();
if (matchs.Count == 0)
{
if (cookie.SeachCount - cookie.WatchCount < 50)
{
Console.WriteLine($"该简历未匹配到公司！ResumeNumber=> {resume.ResumeId}");
resume.Status = 1;
resume.MatchTime = DateTime.Now;
db.SaveChanges();
cookieQueue.Enqueue(cookie);
continue;
}
companys.Add(string.Empty);
}
for (var i = 0; i < matchs.Count; i++)
{
if (i == 2)
{
break;
}
companys.Add(matchs[i].Value);
}
var isMatched = true;
var age = string.Empty;
for (var i = 0; i < companys.Count; i++)
{
var companyName = companys[i];
var statistic = xdb.ZhaopinResumeMatchStatistic.FirstOrDefault(f => f.Date == todayString && f.CompanyId == cookie.CompanyId);
if (statistic == null)
{
continue;
}
var cookieContainer = cookie.Cookie.Serialize(".zhaopin.com");
var start = string.IsNullOrEmpty(companyName) ? startNum * 100 : 0;
var paramDictionary = new Dictionary <string, string>
{
{ "keywords", "的" },
{ "startNum", $"{start}" },
{ "rowsCount", "100" },
{ "sortColumnName", "sortUpDate" },
{ "sortColumn", "sortUpDate desc" },
{ "onlyHasImg", "false" },
{ "anyKeyWord", "false" },
{ "sex", gender },
{ "companyName", companyName },
{ "onlyLastWork", "false" },
{ "ageStart", age },
{ "ageEnd", age }
};
var requestResult = HttpClientFactory.RequestForString("https://ihr.zhaopin.com/resumesearch/search.do", HttpMethod.Post, paramDictionary, cookieContainer);
if (!requestResult.IsSuccess)
{
isMatched = false;
break;
}
var jObject = JsonConvert.DeserializeObject(requestResult.Data) as JObject;
if (jObject["code"] != null)
{
LogFactory.Warn($"CompanyId => {cookie.CompanyId} 搜索异常 异常信息：{(string)jObject["message"]}");
isMatched = false;
return;
}
statistic.SearchCount += 1;
++cookie.SeachCount;
Console.WriteLine($"CompanyId => {cookie.CompanyId} 搜索公司=> {companyName} 性别=> {genderMatch.Value} 今日请求次数：{cookie.SeachCount} ");
xdb.SaveChanges();
var matchResumes = (JArray)jObject?["results"];
if (matchResumes == null)
{
continue;
}
if (matchResumes.Count == 0)
{
continue;
}
var matchResume = matchResumes.FirstOrDefault(a => ((string)a["number"]).Substring(0, 10) == resume.ResumeId);
var resumes = matchResumes.Where(w => DateTime.Parse((string)w["modifyDate"]) > DateTime.Today.AddDays(-2));
foreach (var item in resumes)
{
var wcount = matchResume == null ? cookie.WatchCount : cookie.WatchCount + 1;
if (cookie.SeachCount <= wcount)
{
break;
}
var number = ((string)item["number"]).Substring(0, 10);
if (xdb.ZhaopinResume.Any(a => a.RandomNumber == number))
{
continue;
}
lock (lockObj)
{
if (xdb.ZhaopinWatchedResume.Any(a => a.ResumeNumber == number))
{
continue;
}
requestResult = HttpClientFactory.RequestForString($"http://ihr.zhaopin.com/resumesearch/getresumedetial.do?resumeNo={(string)item["id"]}_1&resumeSource=1&key=&{(string)item["valResumeTimeStr"]}", HttpMethod.Get, null, cookieContainer);
if (requestResult.IsSuccess)
{
var resumeData = JsonConvert.DeserializeObject <dynamic>(requestResult.Data).data;
var resumeDetail = JsonConvert.DeserializeObject(resumeData.detialJSonStr.ToString());
var resumeid = resumeData.resumeId != null ? (int)resumeData.resumeId : resumeDetail.ResumeId != null ? (int)resumeDetail.ResumeId : 0;
statistic.WatchCount += 1;
Console.WriteLine($"CompanyId => {cookie.CompanyId} 查看简历成功！查看简历份数：{++cookie.WatchCount} ResumeNumber => {number}");
xdb.ZhaopinWatchedResume.Add(new ZhaopinWatchedResume
{
Id = resumeid,
ResumeNumber = number,
CompanyId = cookie.CompanyId,
WatchTime = DateTime.UtcNow
});
xdb.SaveChanges();
}
}
}
if (cookie.SeachCount - cookie.WatchCount > 50 && string.IsNullOrEmpty(companyName) && matchResume == null)
{
i--;
if (startNum == 40)
{
if (string.IsNullOrEmpty(age))
{
age = "18";
}
else
{
age = (Convert.ToInt32(age) + 1).ToString();
}
startNum = 0;
}
else
{
startNum++;
}
continue;
}
if (matchResume == null)
{
continue;
}
var resumeNo = (string)matchResume["id"] + "_1";
var resumeNumber = ((string)matchResume["number"]).Substring(0, 10);
if (xdb.ZhaopinResume.Any(a => a.RandomNumber == resumeNumber))
{
continue;
}
var valResumeTimeStr = (string)matchResume["valResumeTimeStr"];
requestResult = HttpClientFactory.RequestForString($"http://ihr.zhaopin.com/resumesearch/getresumedetial.do?resumeNo={resumeNo}&resumeSource=1&key=&{valResumeTimeStr}", HttpMethod.Get, null, cookieContainer);
if (!requestResult.IsSuccess)
{
continue;
}
++cookie.WatchCount;
var resumeJson = JsonConvert.DeserializeObject <dynamic>(requestResult.Data).data;
var detail = JsonConvert.DeserializeObject(resumeJson.detialJSonStr.ToString());
var resumeId = resumeJson.resumeId != null ? (int)resumeJson.resumeId : detail.ResumeId != null ? (int)detail.ResumeId : 0;
if (xdb.ZhaopinMatchedResume.Any(a => a.Id == resumeId))
{
statistic.MatchedCount += 1;
LogFactory.Warn($"匹配到重复简历！ResumeId=>{resumeId} ResumeNumber=>{resumeNumber}");
continue;
}
if (resumeId == 0)
{
LogFactory.Warn($"CompanyId => {cookie.CompanyId} 解析异常！ResumeId 为空, ResumeNumber：{resumeNumber}");
continue;
}
var userId = (int)resumeJson.userDetials.userMasterId;
var user = xdb.ZhaopinUser.FirstOrDefault(f => f.Id == userId);
if (user != null)
{
if (!user.Source.Contains("MANUAL"))
{
user.Id = userId;
user.Source = "XSS";
user.ModifyTime = BaseFanctory.GetTime((string)detail.DateModified).ToUniversalTime();
user.CreateTime = BaseFanctory.GetTime((string)detail.DateCreated).ToUniversalTime();
user.Cellphone = resume.Cellphone;
user.Email = resume.Email;
user.Name = resumeJson.userDetials.userName.ToString();
user.UpdateTime = DateTime.UtcNow;
user.Username = resumeJson.userDetials.email.ToString();
}
}
else
{
xdb.ZhaopinUser.Add(new ZhaopinUser
{
Id = userId,
Source = "XSS",
ModifyTime = BaseFanctory.GetTime((string)detail.DateModified).ToUniversalTime(),
CreateTime = BaseFanctory.GetTime((string)detail.DateCreated).ToUniversalTime(),
Cellphone = resume.Cellphone,
Email = resume.Email,
Name = resumeJson.userDetials.userName.ToString(),
UpdateTime = DateTime.UtcNow
});
}
var resumeEntity = xdb.ZhaopinResume.FirstOrDefault(f => f.Id == resumeId);
if (resumeEntity == null)
{
xdb.ZhaopinResume.Add(new ZhaopinResume
{
Id = resumeId,
RandomNumber = resumeNumber,
UserId = userId,
RefreshTime = BaseFanctory.GetTime((string)detail.DateModified).ToUniversalTime(),
UpdateTime = DateTime.UtcNow,
UserExtId = detail.UserMasterExtId.ToString(),
DeliveryNumber = resumeEntity?.DeliveryNumber,
Source = resumeEntity == null ? "XSS" : resumeEntity.Source
});
}
else
{
resumeEntity.RandomNumber = resumeNumber;
resumeEntity.UserId = userId;
resumeEntity.RefreshTime = BaseFanctory.GetTime((string)detail.DateModified).ToUniversalTime();
resumeEntity.UpdateTime = DateTime.UtcNow;
resumeEntity.UserExtId = detail.UserMasterExtId.ToString();
resumeEntity.DeliveryNumber = resumeEntity?.DeliveryNumber;
resumeEntity.Source = resumeEntity == null ? "XSS" : resumeEntity.Source;
}
var path = $"{ConfigurationManager.AppSettings["Resume.SavePath"]}{DateTime.Now:yyyyMMdd}";
if (!Directory.Exists(path))
{
Directory.CreateDirectory(path);
}
var resumePath = $@"{path}\{resumeId}.json";
resumeJson.userDetials.email = resume.Email;
resumeJson.userDetials.mobilePhone = resume.Cellphone;
File.WriteAllText(resumePath, JsonConvert.SerializeObject(resumeJson));
uploadOssActionBlock.Post(resumePath);
statistic.MatchedCount += 1;
Console.WriteLine($"CompanyId => {cookie.CompanyId} 搜索简历成功！匹配成功 {++cookie.MatchCount} 份， ResumeNumner=>{resumeNumber}，ResumeId=>{resumeId}");
xdb.ZhaopinMatchedResume.Add(new ZhaopinMatchedResume
{
Id = resumeId,
CompanyId = cookie.CompanyId,
MatchTime = DateTime.UtcNow
});
xdb.SaveChanges();
resume.IsMatched = true;
break;
}
if (cookie.SeachCount < cookie.DailySeachCount)
{
cookieQueue.Enqueue(cookie);
}
if (isMatched)
{
resume.Status = 99;
resume.MatchTime = DateTime.Now;
}
db.SaveChanges();
}
}
catch (Exception ex)
{
//while (true)
//{
// if(ex.InnerException == null) break;
// ex = ex.InnerException;
//}
if (cookie.SeachCount < cookie.DailySeachCount)
{
cookieQueue.Enqueue(cookie);
}
LogFactory.Warn($"程序异常, 异常信息：{ex.Message} 堆栈：{ex.StackTrace}");
}
#endregion
}
}
});
}
}
public void Buquan()
{
var queue = new ConcurrentQueue <KeyValuePair <int, KeyValuePair <string, DateTime> > >();
var index = 0;
Task.Run(() =>
{
while (true)
{
KeyValuePair <int, KeyValuePair <string, DateTime> > temp;
if (!queue.TryDequeue(out temp))
{
Thread.Sleep(10);
continue;
}
using (var db = new MangningXssDBEntities())
{
var resume = db.ZhaopinResume.FirstOrDefault(f => f.Id == temp.Key);
if (resume == null)
{
LogFactory.Warn($"找不到简历 ResumeId =>{temp.Key}");
continue;
}
var user = db.ZhaopinUser.FirstOrDefault(f => f.Id == resume.UserId);
if (user == null)
{
LogFactory.Warn($"找不到用户 ResumeId =>{temp.Key} UserId =>{resume.UserId}");
continue;
}
if (string.IsNullOrEmpty(temp.Value.Key) && string.IsNullOrEmpty(user.Cellphone))
{
db.ZhaopinResume.Remove(resume);
db.ZhaopinUser.Remove(user);
Console.WriteLine($"删除成功！ResumeId=>{temp.Key} {++index}/{buquanTotal}");
}
else
{
resume.UserExtId = temp.Value.Key;
user.CreateTime = temp.Value.Value;
Console.WriteLine($"补全成功！{++index}/{buquanTotal}");
}
db.SaveChanges();
}
}
});
using (var db = new MangningXssDBEntities())
{
var idArray = db.ZhaopinResume.Where(w => w.Source == "Watch").Select(s => s.Id).ToArray();
buquanTotal = idArray.Length;
foreach (var businessId in idArray)
{
try
{
var streamContent = newOss.GetObject(newBucket, $"WatchResume/{businessId}").Content;
using (var stream = new MemoryStream())
{
var bytes = new byte[1024];
int len;
while ((len = streamContent.Read(bytes, 0, bytes.Length)) > 0)
{
stream.Write(bytes, 0, len);
}
var jsonContent = Encoding.UTF8.GetString(GZip.Decompress(stream.ToArray()));
var resumeData = JsonConvert.DeserializeObject <dynamic>(jsonContent);
var resumeDetail = JsonConvert.DeserializeObject <dynamic>((string)resumeData.detialJSonStr);
var createDate = BaseFanctory.GetTime((string)resumeDetail.DateCreated).ToUniversalTime();
var extId = resumeDetail.UserMasterExtId.ToString();
queue.Enqueue(new KeyValuePair <int, KeyValuePair <string, DateTime> >(businessId, new KeyValuePair <string, DateTime>(extId, createDate)));
}
}
catch (Exception ex)
{
LogFactory.Warn($"Oss 获取简历异常！ ResumeId => {businessId} 异常原因 => {ex.Message}");
queue.Enqueue(new KeyValuePair <int, KeyValuePair <string, DateTime> >(businessId, new KeyValuePair <string, DateTime>("", DateTime.Now)));
}
}
}
}
private void ImportDodiInfo()
{
var businessManagement = new BusinessManagement();
//var document = new HtmlAgilityPack.HtmlDocument();
//document.LoadHtml(File.ReadAllText(@"D:\Badoucai\Dodi\2363\2363038\Mian_2363038.txt"));
//var userInfo = businessManagement.FormatUserInfomation(document);
//var business = businessManagement.FormatBusiness(document);
var pathQueue = new ConcurrentQueue <string>();
var filePathQueue = new ConcurrentQueue <string>();
var count = 0;
var index = 0;
var pathList = Directory.GetDirectories(@"D:\Badoucai\Dodi\").ToList();
//pathList.Reverse();
pathList.ForEach(t => pathQueue.Enqueue(t));
Task.Run(() =>
{
while (true)
{
if (filePathQueue.Count > 1000)
{
continue;
}
string path;
if (!pathQueue.TryDequeue(out path))
{
continue;
}
var files = Directory.GetFiles(path, "*", SearchOption.AllDirectories);
count += files.Length;
files.ToList().ForEach(t => filePathQueue.Enqueue(t));
}
});
for (var i = 0; i < 16; i++)
{
Task.Run(() =>
{
while (true)
{
string path;
if (!filePathQueue.TryDequeue(out path))
{
continue;
}
try
{
var document = new HtmlAgilityPack.HtmlDocument();
document.LoadHtml(File.ReadAllText(path));
try
{
var userInfo = businessManagement.FormatUserInfomation(document);
var business = businessManagement.FormatBusiness(document);
using (var db = new MangningXssDBEntities())
{
db.DodiUserInfomation.AddOrUpdate(a => a.Id, userInfo);
db.DodiBusiness.AddOrUpdate(a => a.Id, business);
try
{
db.TransactionSaveChanges();
}
catch (Exception ex)
{
while (true)
{
if (ex.InnerException == null)
{
break;
}
ex = ex.InnerException;
}
Program.SetLog(this.tbx_Log, $"多迪信息SaveChanges异常！异常文件路径：{path}, {ex.Message}");
LogFactory.Warn($"多迪信息SaveChanges异常！异常文件路径：{path}, {ex.Message}");
}
}
}
catch (Exception ex)
{
continue;
}
var indexTemp = Interlocked.Increment(ref index);
var destPath = path.Replace("Dodi", "Dodi-Success");
var destDirectoryPath = Path.GetDirectoryName(destPath);
if (!string.IsNullOrEmpty(destDirectoryPath) && !Directory.Exists(destDirectoryPath))
{
Directory.CreateDirectory(destDirectoryPath);
}
if (File.Exists(destPath))
{
File.Delete(destPath);
}
File.Move(path, destPath);
RunInMainthread(() =>
{
Program.SetLog(this.tbx_Log, $"导入成功！进度：{indexTemp}/{count} {Path.GetFileNameWithoutExtension(path)}");
});
}
catch (Exception ex)
{
RunInMainthread(() =>
{
Program.SetLog(this.tbx_Log, $"多迪信息导入异常！异常文件路径：{path}, {ex.Message}");
});
LogFactory.Warn($"多迪信息导入异常！异常文件路径：{path}, {ex.Message}{Environment.NewLine}{ex.StackTrace}");
}
}
});
}
}
