public void Run()
{
// Config encoding, header, cookie, proxy etc... 定义采集的 Site 对象, 设置 Header、Cookie、代理等
//var site = new Site { EncodingName = "UTF-8" };
var site = new Site();
// Add start/feed urls. 添加初始采集链接
var context = new StockLearningEntities();
var stocks = context.Stocks.ToList();
foreach (var stock in stocks)
{
string range = "sh";
if (stock.StockId.StartsWith("0") || stock.StockId.StartsWith("3"))
{
range = "sz";
}
site.AddStartUrl($"http://f9.eastmoney.com/{range}{stock.StockId}.html");
}
DotnetSpider.Core.Spider spider = DotnetSpider.Core.Spider.Create(site,
// use memoery queue scheduler. 使用内存调度
new QueueDuplicateRemovedScheduler(),
// use custmize processor for Processor
new StockJJRPageProcessor())
// use custmize pipeline for Pipeline
.AddPipeline(new StockJJRPipeline());
spider.Downloader = new HttpClientDownloader();
spider.ThreadNum = 1;
spider.EmptySleepTime = 3000;
// Start crawler 启动爬虫
spider.Run();
}
public override void Process(IEnumerable <ResultItems> resultItems, ISpider spider)
{
var context = new StockLearningEntities();
var stocks = context.Stocks.ToList();
Encoding gb2312 = Encoding.GetEncoding("GB2312");
foreach (var resultItem in resultItems)
{
foreach (var result in resultItem.Results)
{
var resultValue = result.Value as List <string>;
foreach (var item in resultValue)
{
string stockValue = Utility.GetTitleContent(item, "a");
//var match = System.Text.RegularExpressions.Regex.Match(stockValue, @"(.*)\((.*)\)");
var match = System.Text.RegularExpressions.Regex.Match(item, @"<a target=""_blank"" href=""(?<href>.*)"">(?<name>.*)\((?<code>.*)\)</a>", RegexOptions.Singleline);
if (match.Success)
{
string name = match.Groups["name"].Value;
string code = match.Groups["code"].Value;
string href = match.Groups["href"].Value;
string encoding = NPinyin.Pinyin.ConvertEncoding(name, Encoding.UTF8, gb2312);
string pinyin = NPinyin.Pinyin.GetInitials(encoding, gb2312);
if (code.StartsWith("6") || code.StartsWith("0") || code.StartsWith("3"))
{
var stock = stocks.FirstOrDefault(s => s.StockId == code);
if (stock != null)
{
if (stock.StockName != name)
{
stock.StockName = name;
}
if (stock.Pinyin != pinyin)
{
stock.Pinyin = pinyin;
}
if (stock.WebAddress != href)
{
stock.WebAddress = href;
}
}
else
{
context.Stocks.Add(new Stock {
StockId = code, StockName = name, Pinyin = pinyin, WebAddress = href
});
}
}
}
}
context.SaveChanges();
}
}
// Other actions like save data to DB. 可以自由实现插入数据库或保存到文件
}
public void SendStock3521()
{
StockLearning.DataContext.StockLearningEntities db = new StockLearningEntities();
decimal YSTZ = 30; // 营业收入同比增长率
decimal XSMLL = 50; // 销售毛利率
decimal YYJLL = 20; // 净利率
decimal ZZCZZL = 0.1M;
var financialReports = (from p in db.StockFinancialReports.Include("Stock")
where p.YSTZ >= YSTZ && p.XSMLL >= XSMLL && p.YYJLL >= YYJLL && p.UpdatedDate == System.DateTime.Today
select p).ToList();
if (financialReports.Any())
{
var mailSetting = db.SystemSettings.FirstOrDefault(s => s.SystemSettingKey == "MailSetting");
if (mailSetting != null)
{
string account = mailSetting.SystemSettingValue1;
string password = mailSetting.SystemSettingValue2;
string mailTemplate = GetEmbeddedResourceFile("HomeOfPandaEyes.StockLearning.Core.Modules.Email.Templates.Stock3521.html");
var users = db.Users.Where(u => u.IsActive == 1);
var config = new TemplateServiceConfiguration();
config.DisableTempFileLocking = true;
config.CachingProvider = new DefaultCachingProvider(t => { });
Engine.Razor = RazorEngineService.Create(config);
foreach (var user in users)
{
try
{
SMTPEmailSender sender = new SMTPEmailSender("smtp.gmail.com", account, password);
sender.AddReceiver(user.Email);
sender.Subject = "黑眼圈之家 - 3521选股";
string content = Engine.Razor.RunCompile(mailTemplate, "templateKey", null, new { UserName = user.DisplayName, Datas = financialReports });
sender.Content = content;
sender.Send();
}
catch (System.Exception ex)
{
LoggingService.Error(null, ex);
}
}
}
}
}
public override void Process(IEnumerable <ResultItems> resultItems, ISpider spider)
{
var context = new StockLearningEntities();
string stockId = string.Empty;
foreach (var resultItem in resultItems)
{
foreach (var result in resultItem.Results)
{
if (result.Key == "StockID")
{
stockId = result.Value as string;
continue;
}
var resultValue = result.Value as DataTable;
if (resultValue.Rows.Count == 0)
{
return;
}
if (!resultValue.Columns.Contains("指标名称"))
{
continue;
}
Logger.Info($"Stock:{stockId}");
var reports = context.StockFinancialReports.Where(f => f.StockId == stockId).ToList();
List <object> reportDates = new List <object>();
List <object> jjls = new List <object>();
foreach (DataRow row in resultValue.Rows)
{
if (string.IsNullOrEmpty(row["指标名称"].ToString()))
{
continue;
}
decimal jlr = 0;
int unit = 0;
string strJLR = row["净利润(元)"].ToString();
if (strJLR.Contains("万"))
{
unit = 10000;
}
if (strJLR.Contains("亿"))
{
unit = 100000000;
}
strJLR = strJLR.Replace("万", string.Empty).Replace("亿", string.Empty);
decimal.TryParse(strJLR, out jlr);
jlr = jlr * unit;
var reportDate = DateTime.ParseExact(row["指标名称"].ToString(), "yy-MM-dd", System.Globalization.CultureInfo.InvariantCulture);
var report = reports.FirstOrDefault(r => r.ReportDate == reportDate);
decimal ZZCZZL = 0;
decimal.TryParse(row["总资产周转率(次)"].ToString(), out ZZCZZL);
if (report != null)
{
if (reportDate.AddYears(3) < DateTime.Now)
{
continue;
}
if (report.JLR == jlr && report.ZZCZZL == ZZCZZL)
{
continue;
}
}
else
{
continue;
}
report.JLR = jlr;
report.YYJLL = report.YS > 0 ? report.JLR / report.YS * 100 : 0;
report.ZZCZZL = ZZCZZL;
}
context.SaveChanges();
}
}
// Other actions like save data to DB. 可以自由实现插入数据库或保存到文件
}
public override void Process(IEnumerable <ResultItems> resultItems, ISpider spider)
{
var context = new StockLearningEntities();
foreach (var resultItem in resultItems)
{
foreach (var result in resultItem.Results)
{
var resultValue = result.Value as DataTable;
if (resultValue.Rows.Count == 0)
{
return;
}
string stockId = resultValue.Rows[0][this.colSECUCODE].ToString();
Logger.Info($"Stock:{stockId}");
var reports = context.StockFinancialReports.Where(f => f.StockId == stockId).ToList();
foreach (DataRow row in resultValue.Rows)
{
var reportDate = Convert.ToDateTime(row[this.colREPORTDATE].ToString());
var report = reports.FirstOrDefault(r => r.ReportDate == reportDate);
if (report != null)
{
if (reportDate.AddYears(3) < DateTime.Now)
{
continue;
}
if (report.YS == Convert.ToDecimal(row[this.colYS]) && report.SJL == Convert.ToDecimal(row[this.colSJL]))
{
continue;
}
}
else
{
report = new StockFinancialReport();
report.StockId = stockId;
report.ReportDate = reportDate;
context.StockFinancialReports.Add(report);
reports.Add(report);
}
report.EPSJB = Convert.ToDecimal(row[this.colEPSJB]);
decimal EPSKCJB = 0;
decimal.TryParse(row[this.colEPSKCJB].ToString(), out EPSKCJB);
report.EPSKCJB = EPSKCJB;
report.YS = Convert.ToDecimal(row[this.colYS]);
report.YSTZ = Convert.ToDecimal(row[this.colYSTZ]);
report.YSHZ = Convert.ToDecimal(row[this.colYSHZ]);
report.SJL = Convert.ToDecimal(row[this.colSJL]);
report.SJLTZ = Convert.ToDecimal(row[this.colSJLTZ]);
report.SJLHZ = Convert.ToDecimal(row[this.colSJLHZ]);
report.BPS = Convert.ToDecimal(row[this.colBPS]);
report.ROEPJ = Convert.ToDecimal(row[this.colROEPJ]);
decimal MGXJJE = 0;
decimal.TryParse(row[this.colMGXJJE].ToString(), out MGXJJE);
report.MGXJJE = MGXJJE;
report.XSMLL = Convert.ToDecimal(row[this.colXSMLL]);
report.LRFP = row[this.colLRFP].ToString();
report.GXL = Convert.ToDecimal(row[this.colGXL]);
report.NoticeDate = string.IsNullOrWhiteSpace(row[this.colNoticeDate].ToString()) ? new Nullable <DateTime>() : Convert.ToDateTime(row[this.colNoticeDate].ToString());
report.UpdatedDate = DateTime.Now;
}
context.SaveChanges();
}
}
// Other actions like save data to DB. 可以自由实现插入数据库或保存到文件
}