示例#1
0
        public void Run()
        {
            // Config encoding, header, cookie, proxy etc... 定义采集的 Site 对象, 设置 Header、Cookie、代理等
            //var site = new Site { EncodingName = "UTF-8" };
            var site = new Site();

            // Add start/feed urls. 添加初始采集链接
            var context = new StockLearningEntities();
            var stocks  = context.Stocks.ToList();


            foreach (var stock in stocks)
            {
                string range = "sh";
                if (stock.StockId.StartsWith("0") || stock.StockId.StartsWith("3"))
                {
                    range = "sz";
                }
                site.AddStartUrl($"http://f9.eastmoney.com/{range}{stock.StockId}.html");
            }

            DotnetSpider.Core.Spider spider = DotnetSpider.Core.Spider.Create(site,
                                              // use memoery queue scheduler. 使用内存调度
                                                                              new QueueDuplicateRemovedScheduler(),
                                              // use custmize processor for  Processor
                                                                              new StockJJRPageProcessor())
                                              // use custmize pipeline for  Pipeline
                                              .AddPipeline(new StockJJRPipeline());
            spider.Downloader     = new HttpClientDownloader();
            spider.ThreadNum      = 1;
            spider.EmptySleepTime = 3000;

            // Start crawler 启动爬虫
            spider.Run();
        }
        public override void Process(IEnumerable <ResultItems> resultItems, ISpider spider)
        {
            var      context = new StockLearningEntities();
            var      stocks  = context.Stocks.ToList();
            Encoding gb2312  = Encoding.GetEncoding("GB2312");

            foreach (var resultItem in resultItems)
            {
                foreach (var result in resultItem.Results)
                {
                    var resultValue = result.Value as List <string>;

                    foreach (var item in resultValue)
                    {
                        string stockValue = Utility.GetTitleContent(item, "a");
                        //var match = System.Text.RegularExpressions.Regex.Match(stockValue, @"(.*)\((.*)\)");
                        var match = System.Text.RegularExpressions.Regex.Match(item, @"<a target=""_blank"" href=""(?<href>.*)"">(?<name>.*)\((?<code>.*)\)</a>", RegexOptions.Singleline);

                        if (match.Success)
                        {
                            string name     = match.Groups["name"].Value;
                            string code     = match.Groups["code"].Value;
                            string href     = match.Groups["href"].Value;
                            string encoding = NPinyin.Pinyin.ConvertEncoding(name, Encoding.UTF8, gb2312);
                            string pinyin   = NPinyin.Pinyin.GetInitials(encoding, gb2312);

                            if (code.StartsWith("6") || code.StartsWith("0") || code.StartsWith("3"))
                            {
                                var stock = stocks.FirstOrDefault(s => s.StockId == code);
                                if (stock != null)
                                {
                                    if (stock.StockName != name)
                                    {
                                        stock.StockName = name;
                                    }
                                    if (stock.Pinyin != pinyin)
                                    {
                                        stock.Pinyin = pinyin;
                                    }
                                    if (stock.WebAddress != href)
                                    {
                                        stock.WebAddress = href;
                                    }
                                }
                                else
                                {
                                    context.Stocks.Add(new Stock {
                                        StockId = code, StockName = name, Pinyin = pinyin, WebAddress = href
                                    });
                                }
                            }
                        }
                    }
                    context.SaveChanges();
                }
            }
            // Other actions like save data to DB. 可以自由实现插入数据库或保存到文件
        }
示例#3
0
        public void SendStock3521()
        {
            StockLearning.DataContext.StockLearningEntities db = new StockLearningEntities();

            decimal YSTZ             = 30; // 营业收入同比增长率
            decimal XSMLL            = 50; // 销售毛利率
            decimal YYJLL            = 20; // 净利率
            decimal ZZCZZL           = 0.1M;
            var     financialReports = (from p in db.StockFinancialReports.Include("Stock")
                                        where p.YSTZ >= YSTZ && p.XSMLL >= XSMLL && p.YYJLL >= YYJLL && p.UpdatedDate == System.DateTime.Today
                                        select p).ToList();


            if (financialReports.Any())
            {
                var mailSetting = db.SystemSettings.FirstOrDefault(s => s.SystemSettingKey == "MailSetting");

                if (mailSetting != null)
                {
                    string account  = mailSetting.SystemSettingValue1;
                    string password = mailSetting.SystemSettingValue2;

                    string mailTemplate = GetEmbeddedResourceFile("HomeOfPandaEyes.StockLearning.Core.Modules.Email.Templates.Stock3521.html");

                    var users  = db.Users.Where(u => u.IsActive == 1);
                    var config = new TemplateServiceConfiguration();
                    config.DisableTempFileLocking = true;
                    config.CachingProvider        = new DefaultCachingProvider(t => { });
                    Engine.Razor = RazorEngineService.Create(config);
                    foreach (var user in users)
                    {
                        try
                        {
                            SMTPEmailSender sender = new SMTPEmailSender("smtp.gmail.com", account, password);
                            sender.AddReceiver(user.Email);
                            sender.Subject = "黑眼圈之家 - 3521选股";

                            string content = Engine.Razor.RunCompile(mailTemplate, "templateKey", null, new { UserName = user.DisplayName, Datas = financialReports });
                            sender.Content = content;
                            sender.Send();
                        }
                        catch (System.Exception ex)
                        {
                            LoggingService.Error(null, ex);
                        }
                    }
                }
            }
        }
示例#4
0
        public override void Process(IEnumerable <ResultItems> resultItems, ISpider spider)
        {
            var    context = new StockLearningEntities();
            string stockId = string.Empty;

            foreach (var resultItem in resultItems)
            {
                foreach (var result in resultItem.Results)
                {
                    if (result.Key == "StockID")
                    {
                        stockId = result.Value as string;
                        continue;
                    }

                    var resultValue = result.Value as DataTable;

                    if (resultValue.Rows.Count == 0)
                    {
                        return;
                    }

                    if (!resultValue.Columns.Contains("指标名称"))
                    {
                        continue;
                    }

                    Logger.Info($"Stock:{stockId}");

                    var           reports     = context.StockFinancialReports.Where(f => f.StockId == stockId).ToList();
                    List <object> reportDates = new List <object>();
                    List <object> jjls        = new List <object>();

                    foreach (DataRow row in resultValue.Rows)
                    {
                        if (string.IsNullOrEmpty(row["指标名称"].ToString()))
                        {
                            continue;
                        }

                        decimal jlr    = 0;
                        int     unit   = 0;
                        string  strJLR = row["净利润(元)"].ToString();
                        if (strJLR.Contains("万"))
                        {
                            unit = 10000;
                        }
                        if (strJLR.Contains("亿"))
                        {
                            unit = 100000000;
                        }
                        strJLR = strJLR.Replace("万", string.Empty).Replace("亿", string.Empty);
                        decimal.TryParse(strJLR, out jlr);
                        jlr = jlr * unit;

                        var reportDate = DateTime.ParseExact(row["指标名称"].ToString(), "yy-MM-dd", System.Globalization.CultureInfo.InvariantCulture);

                        var     report = reports.FirstOrDefault(r => r.ReportDate == reportDate);
                        decimal ZZCZZL = 0;
                        decimal.TryParse(row["总资产周转率(次)"].ToString(), out ZZCZZL);

                        if (report != null)
                        {
                            if (reportDate.AddYears(3) < DateTime.Now)
                            {
                                continue;
                            }
                            if (report.JLR == jlr && report.ZZCZZL == ZZCZZL)
                            {
                                continue;
                            }
                        }
                        else
                        {
                            continue;
                        }
                        report.JLR    = jlr;
                        report.YYJLL  = report.YS > 0 ? report.JLR / report.YS * 100 : 0;
                        report.ZZCZZL = ZZCZZL;
                    }
                    context.SaveChanges();
                }
            }
            // Other actions like save data to DB. 可以自由实现插入数据库或保存到文件
        }
        public override void Process(IEnumerable <ResultItems> resultItems, ISpider spider)
        {
            var context = new StockLearningEntities();

            foreach (var resultItem in resultItems)
            {
                foreach (var result in resultItem.Results)
                {
                    var resultValue = result.Value as DataTable;

                    if (resultValue.Rows.Count == 0)
                    {
                        return;
                    }
                    string stockId = resultValue.Rows[0][this.colSECUCODE].ToString();

                    Logger.Info($"Stock:{stockId}");

                    var reports = context.StockFinancialReports.Where(f => f.StockId == stockId).ToList();
                    foreach (DataRow row in resultValue.Rows)
                    {
                        var reportDate = Convert.ToDateTime(row[this.colREPORTDATE].ToString());

                        var report = reports.FirstOrDefault(r => r.ReportDate == reportDate);
                        if (report != null)
                        {
                            if (reportDate.AddYears(3) < DateTime.Now)
                            {
                                continue;
                            }
                            if (report.YS == Convert.ToDecimal(row[this.colYS]) && report.SJL == Convert.ToDecimal(row[this.colSJL]))
                            {
                                continue;
                            }
                        }
                        else
                        {
                            report            = new StockFinancialReport();
                            report.StockId    = stockId;
                            report.ReportDate = reportDate;
                            context.StockFinancialReports.Add(report);
                            reports.Add(report);
                        }
                        report.EPSJB = Convert.ToDecimal(row[this.colEPSJB]);
                        decimal EPSKCJB = 0;
                        decimal.TryParse(row[this.colEPSKCJB].ToString(), out EPSKCJB);
                        report.EPSKCJB = EPSKCJB;
                        report.YS      = Convert.ToDecimal(row[this.colYS]);
                        report.YSTZ    = Convert.ToDecimal(row[this.colYSTZ]);
                        report.YSHZ    = Convert.ToDecimal(row[this.colYSHZ]);
                        report.SJL     = Convert.ToDecimal(row[this.colSJL]);
                        report.SJLTZ   = Convert.ToDecimal(row[this.colSJLTZ]);
                        report.SJLHZ   = Convert.ToDecimal(row[this.colSJLHZ]);
                        report.BPS     = Convert.ToDecimal(row[this.colBPS]);
                        report.ROEPJ   = Convert.ToDecimal(row[this.colROEPJ]);
                        decimal MGXJJE = 0;
                        decimal.TryParse(row[this.colMGXJJE].ToString(), out MGXJJE);
                        report.MGXJJE      = MGXJJE;
                        report.XSMLL       = Convert.ToDecimal(row[this.colXSMLL]);
                        report.LRFP        = row[this.colLRFP].ToString();
                        report.GXL         = Convert.ToDecimal(row[this.colGXL]);
                        report.NoticeDate  = string.IsNullOrWhiteSpace(row[this.colNoticeDate].ToString()) ? new Nullable <DateTime>() : Convert.ToDateTime(row[this.colNoticeDate].ToString());
                        report.UpdatedDate = DateTime.Now;
                    }
                    context.SaveChanges();
                }
            }
            // Other actions like save data to DB. 可以自由实现插入数据库或保存到文件
        }