public void createJobs() { string StepNamespace = "eqranews.crawling.Models.CrawlSteps"; db.CrawlSources.ToList().ForEach(source => source.CrawlStepper.ToList().ForEach(stepper => { if (stepper.Enabled) { var stepperEng = new eqranews.crawling.Models.CrawlStepper(); List <CrawlResult> results = new List <CrawlResult>(); stepper.CrawlSteps.ToList().ForEach(step => { List <CrawlItem> CrawlItems = new List <CrawlItem>(); var newParams = new object[] { step.Id, CrawlSetpType.Single, new AngleSharp.Url(step.Url), step.Selector }; step.CrawlItems.ToList().ForEach(item => { CrawlItems.Add(new CrawlItem { Name = item.Name, Selector = item.Selector, Attr = item.Attr, Value = item.Value }); }); string StepClassFullName = StepNamespace + "." + step.CrawlStepType.Name; Type t = Type.GetType(StepClassFullName); var instance = (CrawlStep)Activator.CreateInstance(t, newParams); var myPropInfo = t.GetProperty("CrawlItems"); myPropInfo.SetValue(instance, CrawlItems); // var instance = (CrawlStep)Activator.CreateInstance("eqranews.crawling", StepClassFullName, newParams); // MethodInfo method = instance.GetType().GetMethod("Process", BindingFlags.Instance | BindingFlags.Public); // // method.MakeGenericMethod(new[] { t }); // Add step to stepper stepperEng.StepSequence.Add(instance); // results = (List<CrawlResult>)method.Invoke(instance, new object[] { results }); }); results = stepperEng.Crawl(); } }) ); backgroundJobClient.Enqueue(() => Console.WriteLine("Hi from static file CRON JOB!! From :")); recurringJobManager.AddOrUpdate("Run Every Minute", () => Console.WriteLine("Test Recuring Job !!!"), "(*/30 * * * * *"); }
public void CrawlStepperToDatabase(int Id) { var stepperEng = new eqranews.crawling.Models.CrawlStepper(); List <eqranews.crawling.Models.CrawlResult> results = new List <eqranews.crawling.Models.CrawlResult>(); using (var scope = _serviceProvider.CreateScope()) { // var services = _serviceProvider; ApplicationDbContext db = scope.ServiceProvider.GetRequiredService <ApplicationDbContext>(); if (db.CrawlSteppers.Where(x => x.Id == Id).Any()) { DAL.Crawling.CrawlStepper stepper = db.CrawlSteppers.Include(s => s.CrawlSteps).ThenInclude(i => i.CrawlItems).Include("CrawlSteps.CrawlStepType").Where(x => x.Id == Id).SingleOrDefault(); foreach (DAL.Crawling.CrawlStep step in stepper.CrawlSteps.ToList()) { List <eqranews.crawling.Models.CrawlItem> CrawlItems = new List <eqranews.crawling.Models.CrawlItem>(); var newParams = new object[] { step.Id, crawling.Models.CrawlSteps.CrawlSetpType.Single, new AngleSharp.Url(step.Url), step.Selector }; step.CrawlItems.ToList().ForEach(item => { CrawlItems.Add(new eqranews.crawling.Models.CrawlItem { Name = item.Name, Selector = item.Selector, Attr = item.Attr, Value = item.Value }); }); string StepClassFullName = String.Format("eqranews.crawling.Models.CrawlSteps.{0}, eqranews.crawling, Version=1.0.0.0, Culture=neutral, PublicKeyToken=null", step.CrawlStepType.Name); Type t = Type.GetType(StepClassFullName); if (t == null) { return; } var instance = (eqranews.crawling.Models.CrawlSteps.CrawlStep)Activator.CreateInstance(t, newParams); var myPropInfo = t.GetProperty("CrawlItems"); myPropInfo.SetValue(instance, CrawlItems); // Add step to stepper stepperEng.StepSequence.Add(instance); } ; results = stepperEng.Crawl(); SaveToStore(results, db, stepper); } } }