private static void Main(string[] args)
{
string baseUrl = "https://www.joehaydenrealtor.com/louisville-homes/?pg=";
ScraperDAL scraperDal = new ScraperDAL();
try
{
Random random = new Random();
for (int i = 2; i < 10; i++)
{
string url = baseUrl + i;
Task <IHtmlDocument> documentTask = GetHtmlDocument(url);
documentTask.Wait();
var document = documentTask.Result;
List <string> listings = GetListingsFromHtmlDoc(document);
foreach (var listing in listings)
{
scraperDal.SaveDataListingBlob(listing);
}
Thread.Sleep(random.Next(1000, 3000));
}
}
catch (Exception e)
{
var error = new ErrorLog();
error.ApplicationName = "Scraper";
error.ExceptionMessage = e.Message;
error.StackTrace = e.StackTrace;
//scraperDal.SaveErrorLog(error);
}
}
static void Main(string[] args)
{
ScraperDAL scraperDal = new ScraperDAL();
ListingParser parser = new ListingParser();
try
{
var listingBlobs = scraperDal.GetUnprocessedListingBlobs();
List <Listing> parsedListings = parser.ParseListings(listingBlobs);
scraperDal.SaveListings(parsedListings);
//mark as processed
foreach (var listingBlob in listingBlobs)
{
scraperDal.MarkProcessed(listingBlob.ListingBlobId);
}
//after the information is parsed
// mark processed as "1" and then move on
//update the method
}
catch (Exception e)
{
var error = new ErrorLog();
error.ApplicationName = "Scraper";
error.ExceptionMessage = e.Message;
error.StackTrace = e.StackTrace;
scraperDal.SaveErrorLog(error);
}
}
