예제 #1
0
        public void createJobs()
        {
            string StepNamespace = "eqranews.crawling.Models.CrawlSteps";

            db.CrawlSources.ToList().ForEach(source =>
                                             source.CrawlStepper.ToList().ForEach(stepper =>
            {
                if (stepper.Enabled)
                {
                    var stepperEng             = new eqranews.crawling.Models.CrawlStepper();
                    List <CrawlResult> results = new List <CrawlResult>();
                    stepper.CrawlSteps.ToList().ForEach(step =>
                    {
                        List <CrawlItem> CrawlItems = new List <CrawlItem>();

                        var newParams = new object[] { step.Id, CrawlSetpType.Single, new AngleSharp.Url(step.Url), step.Selector };

                        step.CrawlItems.ToList().ForEach(item => {
                            CrawlItems.Add(new CrawlItem {
                                Name = item.Name, Selector = item.Selector, Attr = item.Attr, Value = item.Value
                            });
                        });


                        string StepClassFullName = StepNamespace + "." + step.CrawlStepType.Name;
                        Type t         = Type.GetType(StepClassFullName);
                        var instance   = (CrawlStep)Activator.CreateInstance(t, newParams);
                        var myPropInfo = t.GetProperty("CrawlItems");
                        myPropInfo.SetValue(instance, CrawlItems);

                        // var instance = (CrawlStep)Activator.CreateInstance("eqranews.crawling", StepClassFullName, newParams);
                        // MethodInfo method = instance.GetType().GetMethod("Process", BindingFlags.Instance | BindingFlags.Public);
                        // // method.MakeGenericMethod(new[] { t });


                        // Add step to stepper
                        stepperEng.StepSequence.Add(instance);
                        // results = (List<CrawlResult>)method.Invoke(instance, new object[] { results });
                    });
                    results = stepperEng.Crawl();
                }
            })
                                             );

            backgroundJobClient.Enqueue(() => Console.WriteLine("Hi from static file CRON JOB!! From :"));
            recurringJobManager.AddOrUpdate("Run Every Minute", () => Console.WriteLine("Test Recuring Job !!!"), "(*/30 * * * * *");
        }
        public void CrawlStepperToDatabase(int Id)
        {
            var stepperEng = new eqranews.crawling.Models.CrawlStepper();
            List <eqranews.crawling.Models.CrawlResult> results = new List <eqranews.crawling.Models.CrawlResult>();

            using (var scope = _serviceProvider.CreateScope())
            {
                // var services = _serviceProvider;
                ApplicationDbContext db = scope.ServiceProvider.GetRequiredService <ApplicationDbContext>();
                if (db.CrawlSteppers.Where(x => x.Id == Id).Any())
                {
                    DAL.Crawling.CrawlStepper stepper = db.CrawlSteppers.Include(s => s.CrawlSteps).ThenInclude(i => i.CrawlItems).Include("CrawlSteps.CrawlStepType").Where(x => x.Id == Id).SingleOrDefault();

                    foreach (DAL.Crawling.CrawlStep step in stepper.CrawlSteps.ToList())
                    {
                        List <eqranews.crawling.Models.CrawlItem> CrawlItems = new List <eqranews.crawling.Models.CrawlItem>();

                        var newParams = new object[] { step.Id, crawling.Models.CrawlSteps.CrawlSetpType.Single, new AngleSharp.Url(step.Url), step.Selector };

                        step.CrawlItems.ToList().ForEach(item =>
                        {
                            CrawlItems.Add(new eqranews.crawling.Models.CrawlItem {
                                Name = item.Name, Selector = item.Selector, Attr = item.Attr, Value = item.Value
                            });
                        });

                        string StepClassFullName = String.Format("eqranews.crawling.Models.CrawlSteps.{0}, eqranews.crawling, Version=1.0.0.0, Culture=neutral, PublicKeyToken=null", step.CrawlStepType.Name);
                        Type   t = Type.GetType(StepClassFullName);
                        if (t == null)
                        {
                            return;
                        }

                        var instance   = (eqranews.crawling.Models.CrawlSteps.CrawlStep)Activator.CreateInstance(t, newParams);
                        var myPropInfo = t.GetProperty("CrawlItems");
                        myPropInfo.SetValue(instance, CrawlItems);

                        // Add step to stepper
                        stepperEng.StepSequence.Add(instance);
                    }
                    ;
                    results = stepperEng.Crawl();
                    SaveToStore(results, db, stepper);
                }
            }
        }