public override void Process(IEnumerable <ResultItems> resultItems, ISpider spider) { var context = new StockLearningEntities(); var stocks = context.Stocks.ToList(); Encoding gb2312 = Encoding.GetEncoding("GB2312"); foreach (var resultItem in resultItems) { foreach (var result in resultItem.Results) { var resultValue = result.Value as List <string>; foreach (var item in resultValue) { string stockValue = Utility.GetTitleContent(item, "a"); //var match = System.Text.RegularExpressions.Regex.Match(stockValue, @"(.*)\((.*)\)"); var match = System.Text.RegularExpressions.Regex.Match(item, @"<a target=""_blank"" href=""(?<href>.*)"">(?<name>.*)\((?<code>.*)\)</a>", RegexOptions.Singleline); if (match.Success) { string name = match.Groups["name"].Value; string code = match.Groups["code"].Value; string href = match.Groups["href"].Value; string encoding = NPinyin.Pinyin.ConvertEncoding(name, Encoding.UTF8, gb2312); string pinyin = NPinyin.Pinyin.GetInitials(encoding, gb2312); if (code.StartsWith("6") || code.StartsWith("0") || code.StartsWith("3")) { var stock = stocks.FirstOrDefault(s => s.StockId == code); if (stock != null) { if (stock.StockName != name) { stock.StockName = name; } if (stock.Pinyin != pinyin) { stock.Pinyin = pinyin; } if (stock.WebAddress != href) { stock.WebAddress = href; } } else { context.Stocks.Add(new Stock { StockId = code, StockName = name, Pinyin = pinyin, WebAddress = href }); } } } } context.SaveChanges(); } } // Other actions like save data to DB. 可以自由实现插入数据库或保存到文件 }
public override void Process(IEnumerable <ResultItems> resultItems, ISpider spider) { var context = new StockLearningEntities(); string stockId = string.Empty; foreach (var resultItem in resultItems) { foreach (var result in resultItem.Results) { if (result.Key == "StockID") { stockId = result.Value as string; continue; } var resultValue = result.Value as DataTable; if (resultValue.Rows.Count == 0) { return; } if (!resultValue.Columns.Contains("指标名称")) { continue; } Logger.Info($"Stock:{stockId}"); var reports = context.StockFinancialReports.Where(f => f.StockId == stockId).ToList(); List <object> reportDates = new List <object>(); List <object> jjls = new List <object>(); foreach (DataRow row in resultValue.Rows) { if (string.IsNullOrEmpty(row["指标名称"].ToString())) { continue; } decimal jlr = 0; int unit = 0; string strJLR = row["净利润(元)"].ToString(); if (strJLR.Contains("万")) { unit = 10000; } if (strJLR.Contains("亿")) { unit = 100000000; } strJLR = strJLR.Replace("万", string.Empty).Replace("亿", string.Empty); decimal.TryParse(strJLR, out jlr); jlr = jlr * unit; var reportDate = DateTime.ParseExact(row["指标名称"].ToString(), "yy-MM-dd", System.Globalization.CultureInfo.InvariantCulture); var report = reports.FirstOrDefault(r => r.ReportDate == reportDate); decimal ZZCZZL = 0; decimal.TryParse(row["总资产周转率(次)"].ToString(), out ZZCZZL); if (report != null) { if (reportDate.AddYears(3) < DateTime.Now) { continue; } if (report.JLR == jlr && report.ZZCZZL == ZZCZZL) { continue; } } else { continue; } report.JLR = jlr; report.YYJLL = report.YS > 0 ? report.JLR / report.YS * 100 : 0; report.ZZCZZL = ZZCZZL; } context.SaveChanges(); } } // Other actions like save data to DB. 可以自由实现插入数据库或保存到文件 }
public override void Process(IEnumerable <ResultItems> resultItems, ISpider spider) { var context = new StockLearningEntities(); foreach (var resultItem in resultItems) { foreach (var result in resultItem.Results) { var resultValue = result.Value as DataTable; if (resultValue.Rows.Count == 0) { return; } string stockId = resultValue.Rows[0][this.colSECUCODE].ToString(); Logger.Info($"Stock:{stockId}"); var reports = context.StockFinancialReports.Where(f => f.StockId == stockId).ToList(); foreach (DataRow row in resultValue.Rows) { var reportDate = Convert.ToDateTime(row[this.colREPORTDATE].ToString()); var report = reports.FirstOrDefault(r => r.ReportDate == reportDate); if (report != null) { if (reportDate.AddYears(3) < DateTime.Now) { continue; } if (report.YS == Convert.ToDecimal(row[this.colYS]) && report.SJL == Convert.ToDecimal(row[this.colSJL])) { continue; } } else { report = new StockFinancialReport(); report.StockId = stockId; report.ReportDate = reportDate; context.StockFinancialReports.Add(report); reports.Add(report); } report.EPSJB = Convert.ToDecimal(row[this.colEPSJB]); decimal EPSKCJB = 0; decimal.TryParse(row[this.colEPSKCJB].ToString(), out EPSKCJB); report.EPSKCJB = EPSKCJB; report.YS = Convert.ToDecimal(row[this.colYS]); report.YSTZ = Convert.ToDecimal(row[this.colYSTZ]); report.YSHZ = Convert.ToDecimal(row[this.colYSHZ]); report.SJL = Convert.ToDecimal(row[this.colSJL]); report.SJLTZ = Convert.ToDecimal(row[this.colSJLTZ]); report.SJLHZ = Convert.ToDecimal(row[this.colSJLHZ]); report.BPS = Convert.ToDecimal(row[this.colBPS]); report.ROEPJ = Convert.ToDecimal(row[this.colROEPJ]); decimal MGXJJE = 0; decimal.TryParse(row[this.colMGXJJE].ToString(), out MGXJJE); report.MGXJJE = MGXJJE; report.XSMLL = Convert.ToDecimal(row[this.colXSMLL]); report.LRFP = row[this.colLRFP].ToString(); report.GXL = Convert.ToDecimal(row[this.colGXL]); report.NoticeDate = string.IsNullOrWhiteSpace(row[this.colNoticeDate].ToString()) ? new Nullable <DateTime>() : Convert.ToDateTime(row[this.colNoticeDate].ToString()); report.UpdatedDate = DateTime.Now; } context.SaveChanges(); } } // Other actions like save data to DB. 可以自由实现插入数据库或保存到文件 }