static void Main(string[] args)
{
int i = 0;
var listHotels = new System.Collections.Concurrent.ConcurrentBag<Data>();
/* BAliurl * */
var extUrl = "/RestaurantSearch?ajax=0&geo=294226&Action=PAGE&o=a{0}&etags=9910%2C9911%2C9909%2C9901%2C9899%2C9900";
// var extUrl = "/RestaurantSearch?ajax=0&geo=294265&Action=PAGE&o=a{0}&etags=9909%2C9899%2C9901%2C9900%2C9910%2C9911";
var nods = Enumerable.Range(1, 7110).Where(p => p % 30 == 0);
//var nods = Enumerable.Range(1, 40).Where(p => p % 30 == 0);
Parallel.ForEach(nods, o =>
// {
//});
//for (var o = 0; o <= 3480; )
{
try
{
HtmlDocument doc = null;
for (var ic = 0; ic < 10 && doc == null; ic++)
{
// HtmlWeb web = new HtmlWeb();
// web.PreRequest += new HtmlWeb.PreRequestHandler(onPrereq);
try
{
doc = GetDoc(base_url + string.Format(extUrl, o));
}
catch (Exception e)
{
Console.WriteLine("error to load " + base_url + string.Format(extUrl, o) + " " + ic);
System.Threading.Thread.Sleep(1000);
}
}
//var nodes = doc.DocumentNode.Descendants().Where(p =>x p.Name == "div").Where(o => o.Id.Contains("hotel_")).Where(i => i.Attributes.FirstOrDefault(n => n.Value == "listing_title") != null);//.Where(p => p.Attributes.Contains("class='listing_info popIndexValidation'"));
if (doc != null)
{
var nodes = doc.DocumentNode.Descendants().Where(p => p.Name == "h3" && p.Attributes.FirstOrDefault(n => n.Value == "title") != null);//.Where(p => p.Attributes.Contains("class='listing_info popIndexValidation'"));
// Parallel.ForEach(nodes, item =>
foreach (var item in nodes)
{
var test = item.ChildNodes.Where(p => p.NodeType == HtmlNodeType.Element && p.Name == "a").FirstOrDefault();
if (test != null)
{
var d = new Data();
d.Name = test.InnerText;
var url = test.Attributes.FirstOrDefault(p => p.Name == "href");
if (url != null)
{
d.ID = ++i;
d.url = url.Value;
}
d.ListUrl = base_url + d.url;// "/Restaurants-g294226-Bali.html";
listHotels.Add(d);
LoadReview(d);
WriteOutput(d);
Console.WriteLine(d.ToString());
}
}
//);
}
}
catch { }
// j += 30;
}
);
WriteOutput(listHotels.ToList());
// HtmlNodeCollection tags = doc.DocumentNode.SelectNodes("//abc//tag");
}
