public void createJobs()
{
string StepNamespace = "eqranews.crawling.Models.CrawlSteps";
db.CrawlSources.ToList().ForEach(source =>
source.CrawlStepper.ToList().ForEach(stepper =>
{
if (stepper.Enabled)
{
var stepperEng = new eqranews.crawling.Models.CrawlStepper();
List <CrawlResult> results = new List <CrawlResult>();
stepper.CrawlSteps.ToList().ForEach(step =>
{
List <CrawlItem> CrawlItems = new List <CrawlItem>();
var newParams = new object[] { step.Id, CrawlSetpType.Single, new AngleSharp.Url(step.Url), step.Selector };
step.CrawlItems.ToList().ForEach(item => {
CrawlItems.Add(new CrawlItem {
Name = item.Name, Selector = item.Selector, Attr = item.Attr, Value = item.Value
});
});
string StepClassFullName = StepNamespace + "." + step.CrawlStepType.Name;
Type t = Type.GetType(StepClassFullName);
var instance = (CrawlStep)Activator.CreateInstance(t, newParams);
var myPropInfo = t.GetProperty("CrawlItems");
myPropInfo.SetValue(instance, CrawlItems);
// var instance = (CrawlStep)Activator.CreateInstance("eqranews.crawling", StepClassFullName, newParams);
// MethodInfo method = instance.GetType().GetMethod("Process", BindingFlags.Instance | BindingFlags.Public);
// // method.MakeGenericMethod(new[] { t });
// Add step to stepper
stepperEng.StepSequence.Add(instance);
// results = (List<CrawlResult>)method.Invoke(instance, new object[] { results });
});
results = stepperEng.Crawl();
}
})
);
backgroundJobClient.Enqueue(() => Console.WriteLine("Hi from static file CRON JOB!! From :"));
recurringJobManager.AddOrUpdate("Run Every Minute", () => Console.WriteLine("Test Recuring Job !!!"), "(*/30 * * * * *");
}
public void CrawlStepperToDatabase(int Id)
{
var stepperEng = new eqranews.crawling.Models.CrawlStepper();
List <eqranews.crawling.Models.CrawlResult> results = new List <eqranews.crawling.Models.CrawlResult>();
using (var scope = _serviceProvider.CreateScope())
{
// var services = _serviceProvider;
ApplicationDbContext db = scope.ServiceProvider.GetRequiredService <ApplicationDbContext>();
if (db.CrawlSteppers.Where(x => x.Id == Id).Any())
{
DAL.Crawling.CrawlStepper stepper = db.CrawlSteppers.Include(s => s.CrawlSteps).ThenInclude(i => i.CrawlItems).Include("CrawlSteps.CrawlStepType").Where(x => x.Id == Id).SingleOrDefault();
foreach (DAL.Crawling.CrawlStep step in stepper.CrawlSteps.ToList())
{
List <eqranews.crawling.Models.CrawlItem> CrawlItems = new List <eqranews.crawling.Models.CrawlItem>();
var newParams = new object[] { step.Id, crawling.Models.CrawlSteps.CrawlSetpType.Single, new AngleSharp.Url(step.Url), step.Selector };
step.CrawlItems.ToList().ForEach(item =>
{
CrawlItems.Add(new eqranews.crawling.Models.CrawlItem {
Name = item.Name, Selector = item.Selector, Attr = item.Attr, Value = item.Value
});
});
string StepClassFullName = String.Format("eqranews.crawling.Models.CrawlSteps.{0}, eqranews.crawling, Version=1.0.0.0, Culture=neutral, PublicKeyToken=null", step.CrawlStepType.Name);
Type t = Type.GetType(StepClassFullName);
if (t == null)
{
return;
}
var instance = (eqranews.crawling.Models.CrawlSteps.CrawlStep)Activator.CreateInstance(t, newParams);
var myPropInfo = t.GetProperty("CrawlItems");
myPropInfo.SetValue(instance, CrawlItems);
// Add step to stepper
stepperEng.StepSequence.Add(instance);
}
;
results = stepperEng.Crawl();
SaveToStore(results, db, stepper);
}
}
}
