public override void Process(IEnumerable <ResultItems> resultItems, ISpider spider)
{
var context = new StockLearningEntities();
var stocks = context.Stocks.ToList();
Encoding gb2312 = Encoding.GetEncoding("GB2312");
foreach (var resultItem in resultItems)
{
foreach (var result in resultItem.Results)
{
var resultValue = result.Value as List <string>;
foreach (var item in resultValue)
{
string stockValue = Utility.GetTitleContent(item, "a");
//var match = System.Text.RegularExpressions.Regex.Match(stockValue, @"(.*)\((.*)\)");
var match = System.Text.RegularExpressions.Regex.Match(item, @"<a target=""_blank"" href=""(?<href>.*)"">(?<name>.*)\((?<code>.*)\)</a>", RegexOptions.Singleline);
if (match.Success)
{
string name = match.Groups["name"].Value;
string code = match.Groups["code"].Value;
string href = match.Groups["href"].Value;
string encoding = NPinyin.Pinyin.ConvertEncoding(name, Encoding.UTF8, gb2312);
string pinyin = NPinyin.Pinyin.GetInitials(encoding, gb2312);
if (code.StartsWith("6") || code.StartsWith("0") || code.StartsWith("3"))
{
var stock = stocks.FirstOrDefault(s => s.StockId == code);
if (stock != null)
{
if (stock.StockName != name)
{
stock.StockName = name;
}
if (stock.Pinyin != pinyin)
{
stock.Pinyin = pinyin;
}
if (stock.WebAddress != href)
{
stock.WebAddress = href;
}
}
else
{
context.Stocks.Add(new Stock {
StockId = code, StockName = name, Pinyin = pinyin, WebAddress = href
});
}
}
}
}
context.SaveChanges();
}
}
// Other actions like save data to DB. 可以自由实现插入数据库或保存到文件
}
public override void Process(IEnumerable <ResultItems> resultItems, ISpider spider)
{
var context = new StockLearningEntities();
string stockId = string.Empty;
foreach (var resultItem in resultItems)
{
foreach (var result in resultItem.Results)
{
if (result.Key == "StockID")
{
stockId = result.Value as string;
continue;
}
var resultValue = result.Value as DataTable;
if (resultValue.Rows.Count == 0)
{
return;
}
if (!resultValue.Columns.Contains("指标名称"))
{
continue;
}
Logger.Info($"Stock:{stockId}");
var reports = context.StockFinancialReports.Where(f => f.StockId == stockId).ToList();
List <object> reportDates = new List <object>();
List <object> jjls = new List <object>();
foreach (DataRow row in resultValue.Rows)
{
if (string.IsNullOrEmpty(row["指标名称"].ToString()))
{
continue;
}
decimal jlr = 0;
int unit = 0;
string strJLR = row["净利润(元)"].ToString();
if (strJLR.Contains("万"))
{
unit = 10000;
}
if (strJLR.Contains("亿"))
{
unit = 100000000;
}
strJLR = strJLR.Replace("万", string.Empty).Replace("亿", string.Empty);
decimal.TryParse(strJLR, out jlr);
jlr = jlr * unit;
var reportDate = DateTime.ParseExact(row["指标名称"].ToString(), "yy-MM-dd", System.Globalization.CultureInfo.InvariantCulture);
var report = reports.FirstOrDefault(r => r.ReportDate == reportDate);
decimal ZZCZZL = 0;
decimal.TryParse(row["总资产周转率(次)"].ToString(), out ZZCZZL);
if (report != null)
{
if (reportDate.AddYears(3) < DateTime.Now)
{
continue;
}
if (report.JLR == jlr && report.ZZCZZL == ZZCZZL)
{
continue;
}
}
else
{
continue;
}
report.JLR = jlr;
report.YYJLL = report.YS > 0 ? report.JLR / report.YS * 100 : 0;
report.ZZCZZL = ZZCZZL;
}
context.SaveChanges();
}
}
// Other actions like save data to DB. 可以自由实现插入数据库或保存到文件
}
public override void Process(IEnumerable <ResultItems> resultItems, ISpider spider)
{
var context = new StockLearningEntities();
foreach (var resultItem in resultItems)
{
foreach (var result in resultItem.Results)
{
var resultValue = result.Value as DataTable;
if (resultValue.Rows.Count == 0)
{
return;
}
string stockId = resultValue.Rows[0][this.colSECUCODE].ToString();
Logger.Info($"Stock:{stockId}");
var reports = context.StockFinancialReports.Where(f => f.StockId == stockId).ToList();
foreach (DataRow row in resultValue.Rows)
{
var reportDate = Convert.ToDateTime(row[this.colREPORTDATE].ToString());
var report = reports.FirstOrDefault(r => r.ReportDate == reportDate);
if (report != null)
{
if (reportDate.AddYears(3) < DateTime.Now)
{
continue;
}
if (report.YS == Convert.ToDecimal(row[this.colYS]) && report.SJL == Convert.ToDecimal(row[this.colSJL]))
{
continue;
}
}
else
{
report = new StockFinancialReport();
report.StockId = stockId;
report.ReportDate = reportDate;
context.StockFinancialReports.Add(report);
reports.Add(report);
}
report.EPSJB = Convert.ToDecimal(row[this.colEPSJB]);
decimal EPSKCJB = 0;
decimal.TryParse(row[this.colEPSKCJB].ToString(), out EPSKCJB);
report.EPSKCJB = EPSKCJB;
report.YS = Convert.ToDecimal(row[this.colYS]);
report.YSTZ = Convert.ToDecimal(row[this.colYSTZ]);
report.YSHZ = Convert.ToDecimal(row[this.colYSHZ]);
report.SJL = Convert.ToDecimal(row[this.colSJL]);
report.SJLTZ = Convert.ToDecimal(row[this.colSJLTZ]);
report.SJLHZ = Convert.ToDecimal(row[this.colSJLHZ]);
report.BPS = Convert.ToDecimal(row[this.colBPS]);
report.ROEPJ = Convert.ToDecimal(row[this.colROEPJ]);
decimal MGXJJE = 0;
decimal.TryParse(row[this.colMGXJJE].ToString(), out MGXJJE);
report.MGXJJE = MGXJJE;
report.XSMLL = Convert.ToDecimal(row[this.colXSMLL]);
report.LRFP = row[this.colLRFP].ToString();
report.GXL = Convert.ToDecimal(row[this.colGXL]);
report.NoticeDate = string.IsNullOrWhiteSpace(row[this.colNoticeDate].ToString()) ? new Nullable <DateTime>() : Convert.ToDateTime(row[this.colNoticeDate].ToString());
report.UpdatedDate = DateTime.Now;
}
context.SaveChanges();
}
}
// Other actions like save data to DB. 可以自由实现插入数据库或保存到文件
}