public CrownePlaza(Scraper.Scraper Scraper)
{
this.Scraper = Scraper;
this.Site = new Site(new Uri("https://www.crowneplaza.com"));
this.Site.OutputType = PipelineOutput.Object;
}
public Expedia(Scraper.Scraper scraper)
{
this.Scraper = scraper;
this.Site = new Site(new Uri("https://www.expedia.co.uk"));
this.Site.OutputType = PipelineOutput.Object;
}
public IHG(Scraper.Scraper scraper)
{
this.Scraper = scraper;
this.Site = new Site(new Uri("https://www.ihg.com/"));
this.Site.OutputType = PipelineOutput.Object;
}
public HolidayInn(Scraper.Scraper Scraper)
{
this.Scraper = Scraper;
this.Site = new Site(new Uri("https://www.holidayinn.com/"));
this.Site.OutputType = PipelineOutput.Object;
}
public Booking(Scraper.Scraper Scraper)
{
this.Scraper = Scraper;
this.Site = new Site(new Uri("https://www.booking.com/"));
this.Site.OutputType = PipelineOutput.Object;
}
public HoteldotCom(Scraper.Scraper Scraper)
{
this.Scraper = Scraper;
this.Site = new Site(new Uri("https://uk.hotels.com/"));
this.Site.OutputType = PipelineOutput.Object;
}
public override void Invoke(CommandArguments commandArgument)
{
Console.WriteLine("Hello World!");
Console.SetWindowSize(Console.LargestWindowWidth / 2, Console.LargestWindowHeight);
Console.SetWindowPosition(0, 0);
var scraper = new Scraper.Scraper();
var ihg = new Site(new Uri("https://www.ihg.com/"));
var london = ihg.AddPage("hotels/gb/en/find-hotels/hotel/list?qDest=London,%20United%20Kingdom&qCiMy=32019&qCiD=16&qCoMy=32019&qCoD=17&qAdlt=1&qChld=0&qRms=1&qRtP=6CBARC&qAkamaiCC=GB&qSrt=sDD&qBrs=re.ic.in.vn.cp.vx.hi.ex.rs.cv.sb.cw.ma.ul.ki.va&srb_u=0&qRad=30&qRdU=mi", By.TagName("body"));
london.AddNode(new NodeRequest {
Property = "Name", XPath = "//div/hotel-row/div[1]/div/div/hotel-details/div/div/div[1]/div/div/span[1]/a"
});
london.AddNode(new NodeRequest {
Property = "Location", XPath = "//div/hotel-row/div[1]/div/div/hotel-details/div/div/div[1]/div/div/span[2]/a"
});
london.AddNode(new NodeRequest {
Property = "Address", XPath = "//div/hotel-row/div[1]/div/div/hotel-details/div/div/div[2]/div[1]/hotel-address/div/div/span[1]"
});
london.AddNode(new NodeRequest {
Property = "Postcode", XPath = "//div/hotel-row/div[1]/div/div/hotel-details/div/div/div[2]/div[1]/hotel-address/div/div/span[1]"
});
london.AddNode(new NodeRequest {
Property = "Country", XPath = "//div/hotel-row/div[1]/div/div/hotel-details/div/div/div[2]/div[1]/hotel-address/div/div/span[3]/span[2]"
});
Site dotnetperls = new Site(new Uri("https://www.dotnetperls.com"));
dotnetperls.OutputType = PipelineOutput.Object;
//PageLayout indexLayout = dotnetperls.AddPage("", By.TagName("body"));
PageLayout asyncLayout = dotnetperls.AddPage("async", By.TagName("body"));
//indexLayout.AddNode(new NodeRequest { Property = "Pages", XPath = "//body//a//b" });
//indexLayout.AddNode(new NodeRequest { Property = "Description", XPath = "//body//a//div" });
asyncLayout.AddNode(new NodeRequest
{
Property = "Title", XPath = "//body//a"
});
Console.WriteLine(ihg.ToString());
scraper.AddSite(ihg);
scraper.AddSite(dotnetperls);
//scraper.RunAll();
scraper.Run(dotnetperls);
//Storage storage = new Storage("local.db");
Console.ReadKey();
}
public void Setup()
{
this.scraper = new Scraper.Scraper();
this.scraper.Links = new Uri[]
{
new Uri(Directory.GetCurrentDirectory() + @"\Sample.html"),
new Uri(Directory.GetCurrentDirectory() + @"\Sample2.html"),
new Uri(Directory.GetCurrentDirectory() + @"\Sample3.html")
};
this.scraper.Download();
}
private static void LoadShows(Scraper scraper)
{
while (true)
{
UpdateShows(scraper);
List<Show> shows;
Logger.Trace($"Retrieving new episodes from {scraper.SiteTitle}");
try
{
shows = scraper.Load();
}
catch (Exception e)
{
Logger.Error(e, $"An error occurred while retrieving new episodes from {scraper.SiteTitle}");
shows = new List<Show>();
}
UpdateEpisodes(shows);
Thread.Sleep(TimeSpan.FromMinutes(updateInterval));
}
}
static void Main(string[] args)
{
var scraper = new Scraper();
var spider = new Spider();
scraper.Links = spider.GetLinks("discover/advanced?category_id=12&woe_id=0&sort=most_funded");
scraper.Links = scraper.Links;//.Take(10); //Todo: add top x option
scraper.Download();
Console.WriteLine("Downloaded and scraped:");
foreach (var project in scraper.Projects)
{
Console.WriteLine(project.Name);
}
Console.WriteLine("Please Enter Output file name:");
var filename = Console.ReadLine();
scraper.Save(new XML(), filename + ".xml");
scraper.Save(new CSV(), filename + ".csv");
}
private static void UpdateShows(Scraper scraper)
{
using (var db = new AppDbContext())
{
List<Show> showsList;
Logger.Debug($"Retrieving shows from {scraper.SiteTitle}");
try
{
showsList = scraper.LoadShows().Result;
}
catch (Exception e)
{
Logger.Error(e, $"An error occurred while retrieving shows from {scraper.SiteTitle}");
return;
}
foreach (var show in showsList)
{
var dbShow = db.SiteTypes.First(st => st.Id == scraper.ShowsSiteType.Id).Shows.FirstOrDefault(s => s.SiteId == show.SiteId);
if (dbShow != null)
{
dbShow.OriginalTitle = show.OriginalTitle;
dbShow.Description = show.Description ?? dbShow.Description;
dbShow.SiteId = show.SiteId;
}
else
{
db.Shows.Add(show);
}
}
db.SaveChanges();
}
}
public SiteManager()
{
scraper = new Scraper.Scraper();
}
