public CrownePlaza(Scraper.Scraper Scraper) { this.Scraper = Scraper; this.Site = new Site(new Uri("https://www.crowneplaza.com")); this.Site.OutputType = PipelineOutput.Object; }
public Expedia(Scraper.Scraper scraper) { this.Scraper = scraper; this.Site = new Site(new Uri("https://www.expedia.co.uk")); this.Site.OutputType = PipelineOutput.Object; }
public IHG(Scraper.Scraper scraper) { this.Scraper = scraper; this.Site = new Site(new Uri("https://www.ihg.com/")); this.Site.OutputType = PipelineOutput.Object; }
public HolidayInn(Scraper.Scraper Scraper) { this.Scraper = Scraper; this.Site = new Site(new Uri("https://www.holidayinn.com/")); this.Site.OutputType = PipelineOutput.Object; }
public Booking(Scraper.Scraper Scraper) { this.Scraper = Scraper; this.Site = new Site(new Uri("https://www.booking.com/")); this.Site.OutputType = PipelineOutput.Object; }
public HoteldotCom(Scraper.Scraper Scraper) { this.Scraper = Scraper; this.Site = new Site(new Uri("https://uk.hotels.com/")); this.Site.OutputType = PipelineOutput.Object; }
public override void Invoke(CommandArguments commandArgument) { Console.WriteLine("Hello World!"); Console.SetWindowSize(Console.LargestWindowWidth / 2, Console.LargestWindowHeight); Console.SetWindowPosition(0, 0); var scraper = new Scraper.Scraper(); var ihg = new Site(new Uri("https://www.ihg.com/")); var london = ihg.AddPage("hotels/gb/en/find-hotels/hotel/list?qDest=London,%20United%20Kingdom&qCiMy=32019&qCiD=16&qCoMy=32019&qCoD=17&qAdlt=1&qChld=0&qRms=1&qRtP=6CBARC&qAkamaiCC=GB&qSrt=sDD&qBrs=re.ic.in.vn.cp.vx.hi.ex.rs.cv.sb.cw.ma.ul.ki.va&srb_u=0&qRad=30&qRdU=mi", By.TagName("body")); london.AddNode(new NodeRequest { Property = "Name", XPath = "//div/hotel-row/div[1]/div/div/hotel-details/div/div/div[1]/div/div/span[1]/a" }); london.AddNode(new NodeRequest { Property = "Location", XPath = "//div/hotel-row/div[1]/div/div/hotel-details/div/div/div[1]/div/div/span[2]/a" }); london.AddNode(new NodeRequest { Property = "Address", XPath = "//div/hotel-row/div[1]/div/div/hotel-details/div/div/div[2]/div[1]/hotel-address/div/div/span[1]" }); london.AddNode(new NodeRequest { Property = "Postcode", XPath = "//div/hotel-row/div[1]/div/div/hotel-details/div/div/div[2]/div[1]/hotel-address/div/div/span[1]" }); london.AddNode(new NodeRequest { Property = "Country", XPath = "//div/hotel-row/div[1]/div/div/hotel-details/div/div/div[2]/div[1]/hotel-address/div/div/span[3]/span[2]" }); Site dotnetperls = new Site(new Uri("https://www.dotnetperls.com")); dotnetperls.OutputType = PipelineOutput.Object; //PageLayout indexLayout = dotnetperls.AddPage("", By.TagName("body")); PageLayout asyncLayout = dotnetperls.AddPage("async", By.TagName("body")); //indexLayout.AddNode(new NodeRequest { Property = "Pages", XPath = "//body//a//b" }); //indexLayout.AddNode(new NodeRequest { Property = "Description", XPath = "//body//a//div" }); asyncLayout.AddNode(new NodeRequest { Property = "Title", XPath = "//body//a" }); Console.WriteLine(ihg.ToString()); scraper.AddSite(ihg); scraper.AddSite(dotnetperls); //scraper.RunAll(); scraper.Run(dotnetperls); //Storage storage = new Storage("local.db"); Console.ReadKey(); }
public void Setup() { this.scraper = new Scraper.Scraper(); this.scraper.Links = new Uri[] { new Uri(Directory.GetCurrentDirectory() + @"\Sample.html"), new Uri(Directory.GetCurrentDirectory() + @"\Sample2.html"), new Uri(Directory.GetCurrentDirectory() + @"\Sample3.html") }; this.scraper.Download(); }
private static void LoadShows(Scraper scraper) { while (true) { UpdateShows(scraper); List<Show> shows; Logger.Trace($"Retrieving new episodes from {scraper.SiteTitle}"); try { shows = scraper.Load(); } catch (Exception e) { Logger.Error(e, $"An error occurred while retrieving new episodes from {scraper.SiteTitle}"); shows = new List<Show>(); } UpdateEpisodes(shows); Thread.Sleep(TimeSpan.FromMinutes(updateInterval)); } }
static void Main(string[] args) { var scraper = new Scraper(); var spider = new Spider(); scraper.Links = spider.GetLinks("discover/advanced?category_id=12&woe_id=0&sort=most_funded"); scraper.Links = scraper.Links;//.Take(10); //Todo: add top x option scraper.Download(); Console.WriteLine("Downloaded and scraped:"); foreach (var project in scraper.Projects) { Console.WriteLine(project.Name); } Console.WriteLine("Please Enter Output file name:"); var filename = Console.ReadLine(); scraper.Save(new XML(), filename + ".xml"); scraper.Save(new CSV(), filename + ".csv"); }
private static void UpdateShows(Scraper scraper) { using (var db = new AppDbContext()) { List<Show> showsList; Logger.Debug($"Retrieving shows from {scraper.SiteTitle}"); try { showsList = scraper.LoadShows().Result; } catch (Exception e) { Logger.Error(e, $"An error occurred while retrieving shows from {scraper.SiteTitle}"); return; } foreach (var show in showsList) { var dbShow = db.SiteTypes.First(st => st.Id == scraper.ShowsSiteType.Id).Shows.FirstOrDefault(s => s.SiteId == show.SiteId); if (dbShow != null) { dbShow.OriginalTitle = show.OriginalTitle; dbShow.Description = show.Description ?? dbShow.Description; dbShow.SiteId = show.SiteId; } else { db.Shows.Add(show); } } db.SaveChanges(); } }
public SiteManager() { scraper = new Scraper.Scraper(); }