public static int ScrapeOlxURLByCounty()
{
int NewURLsCount = 0;
foreach (string judet in Constants.Counties)
{
Log.Information("Scraping URLs for " + judet);
for (int page = 2; page <= 500; page++)
{
string url = string.Format(Constants.OlxBaseSearchUrlByCounty, judet, page);
HashSet <string> urls = new HashSet <string>();
HttpStatusCode status = HttpStatusCode.NotFound;
string html = HttpUtils.DownloadPage(url, ref status);
if (status != HttpStatusCode.OK)
{
break;
}
HtmlAgilityPack.HtmlDocument htmlDoc = new HtmlAgilityPack.HtmlDocument();
htmlDoc.OptionFixNestedTags = true;
htmlDoc.LoadHtml(html);
// ParseErrors is an ArrayList containing any errors from the Load statement
if (htmlDoc.ParseErrors != null && htmlDoc.ParseErrors.Count() > 0)
{
Log.Debug("Error trying to parse content " + htmlDoc.ParseErrors);
Console.WriteLine("Error trying to parse content " + htmlDoc.ParseErrors);
}
else
{
if (htmlDoc.DocumentNode != null)
{
var nodes = htmlDoc.DocumentNode.SelectNodes("//a");
var links = nodes.Where(n => n.HasClass("detailsLink") || n.HasClass("detailsLinkPromoted")).Distinct();
foreach (var link in links)
{
string cleanedURL = HtmlUtils.RemoveAnchorFromURL(link.GetAttributeValue("href", null));
urls.Add(cleanedURL);
}
}
}
PersistenceServices.InsertUrls(urls);
}
}
return(NewURLsCount);
}
void Start()
{
// services created at runtime as managers are also only available at runtime
gridServices = new GridServices();
itemServices = new ItemServices();
carrierServices = new CarrierServices();
buildingServices = new BuildingServices();
assignmentServices = new AssignmentServices();
persistenceServices = new PersistenceServices();
oscsr = GameObject.FindWithTag("OSC").GetComponent <OSCSendReceive>();
soundServices = new SoundServices(oscsr);
EventServices.Instance.SetSoundServices(soundServices);
persistenceServices.InitScene();
}
