public static async Task ParseRouteAsync(Route inputRoute)
{
Console.WriteLine($"Current Route: {inputRoute.Name}");
Stopwatch routeStopwatch = Stopwatch.StartNew();
IHtmlDocument doc = await Utilities.GetHtmlDocAsync(inputRoute.URL);
if (string.IsNullOrEmpty(inputRoute.Name))
{
inputRoute.Name = ParseName(doc);
}
inputRoute.Types = ParseRouteTypes(doc);
inputRoute.Popularity = ParsePopularity(doc);
inputRoute.Rating = ParseRouteRating(doc);
inputRoute.Grades = ParseRouteGrades(doc);
string additionalInfo = ParseAdditionalRouteInfo(doc);
inputRoute.Height = ParseRouteHeight(ref additionalInfo);
inputRoute.AdditionalInfo = additionalInfo;
inputRoute.ParentUrls = GetParentUrls(doc);
doc.Dispose();
Console.WriteLine($"Done with Route: {inputRoute.Name} ({routeStopwatch.Elapsed})");
}
public static List <Area> GetDestAreas()
{
List <Area> destAreas = new List <Area>();
IHtmlDocument doc = Utilities.GetHtmlDoc(Utilities.ALLLOCATIONSURL);
List <IElement> destAreaNodes = doc.GetElementsByTagName("a").Where(x => x.Attributes["href"] != null &&
Utilities.MatchesStateUrlRegex(x.Attributes["href"].Value)).ToList();
destAreaNodes = (from s in destAreaNodes
orderby s.TextContent
group s by s.Attributes["href"].Value into g
select g.First()).ToList();
//Move international to the end
IElement internationalArea = destAreaNodes.Find(p => p.TextContent == "International");
destAreaNodes.Remove(internationalArea);
destAreaNodes.Add(internationalArea);
//Convert to DestArea objects
foreach (IElement destAreaElement in destAreaNodes)
{
Area destArea = new Area()
{
Name = destAreaElement.TextContent,
URL = destAreaElement.Attributes["href"].Value
};
destAreas.Add(destArea);
}
doc.Dispose();
return(destAreas);
}
public static async Task ParseAreaAsync(Area inputArea, bool recursive = true)
{
Console.WriteLine($"Current Area: {inputArea.Name}");
Stopwatch areaStopwatch = Stopwatch.StartNew();
IHtmlDocument doc = await Utilities.GetHtmlDocAsync(inputArea.URL);
if (string.IsNullOrEmpty(inputArea.Name))
{
inputArea.Name = ParseName(doc);
}
inputArea.Statistics = PopulateStatistics(doc);
inputArea.Popularity = ParsePopularity(doc);
inputArea.ParentUrls = GetParentUrls(doc);
inputArea.PopularRouteUrls = GetPopularRouteUrls(doc, 3);
//Get Area's routes
IElement routesTable = doc.GetElementsByTagName("table").Where(p => p.Attributes["id"] != null && p.Attributes["id"].Value == "left-nav-route-table").FirstOrDefault();
List <IElement> htmlRoutes = routesTable == null ? new List <IElement>() : routesTable.GetElementsByTagName("a").ToList();
//Get Area's areas
IElement leftColumnDiv = doc.GetElementsByTagName("div").FirstOrDefault(p => p.Attributes["class"] != null && p.Attributes["class"].Value == "mp-sidebar");
List <IElement> htmlSubAreas = doc.GetElementsByTagName("a").Where(p => p.ParentElement.ParentElement.ParentElement == leftColumnDiv).ToList();
htmlSubAreas.RemoveAll(p => p.ParentElement.ParentElement.Attributes["id"] != null && p.ParentElement.ParentElement.Attributes["id"].Value == "nearbyMTBRides");
htmlSubAreas.RemoveAll(p => !p.Attributes["href"].Value.Contains(Utilities.MPBASEURL));
//Dispose doc
doc.Dispose();
//Populate route details
foreach (IElement routeElement in htmlRoutes)
{
Route route = new Route(routeElement.TextContent, routeElement.Attributes["href"].Value);
inputArea.Routes.Add(route);
TotalRoutes++;
await ParseRouteAsync(route); //Parse route
}
//Populate sub area details
foreach (IElement areaElement in htmlSubAreas)
{
Area subArea = new Area()
{
Name = areaElement.TextContent, URL = areaElement.Attributes["href"].Value
};
inputArea.SubAreas.Add(subArea);
TotalAreas++;
if (recursive)
{
await ParseAreaAsync(subArea); //Parse sub area
}
}
Console.WriteLine($"Done with Area: {inputArea.Name} ({areaStopwatch.Elapsed}). {htmlRoutes.Count} routes, {htmlSubAreas.Count} subareas");
}
private CityVO getCityVO(HtmlParser parser, string dataHtml, string indexCity)
{
IHtmlDocument doc = parser.Parse(dataHtml);
WeatherVO weatherVO;
CityVO cityVO = new CityVO();
cityVO.IndexCity = indexCity;
cityVO.NameCity = doc.QuerySelector("#weather > div.fcontent > div.section.higher > h2").InnerHtml;
cityVO.ImgSrcWeather = doc.QuerySelector("#tab_wdaily2 > img").GetAttribute("src");
cityVO.Day = doc.QuerySelector("#tab_wdaily2 > dl > dd").InnerHtml;
cityVO.DayWeek = doc.QuerySelector("#tab_wdaily2 > dl > dt").InnerHtml;
cityVO.TemperatureMin = doc.QuerySelector("#tab_wdaily2 > div > span.value.m_temp.c").InnerHtml;
cityVO.TemperatureMax = doc.QuerySelector("#tab_wdaily2 > div > em > span.value.m_temp.c").InnerHtml;
WeatherVO[] weather = new WeatherVO[4];
for (int i = 0; i < 4; i++)
{
weatherVO = new WeatherVO();
weatherVO.TimeOfDay = doc.QuerySelector("#tbwdaily2 > tr:nth-child(" + (i + 1) + ") > th:nth-child(1)").InnerHtml.ToString().Trim();
weatherVO.ImgSrcWeather = doc.QuerySelector("#tbwdaily2 > tr:nth-child(" + (i + 1) + ") > td:nth-child(2) > img:nth-child(1)").GetAttribute("src");
weatherVO.FeatureWeather = doc.QuerySelector("#tbwdaily2 > tr:nth-child(" + (i + 1) + ") > td:nth-child(3)").InnerHtml;
weatherVO.TemperatureAir = doc.QuerySelector("#tbwdaily2 > tr:nth-child(" + (i + 1) + ") > td:nth-child(4) > span:nth-child(1)").InnerHtml;
weatherVO.Pressure = doc.QuerySelector("#tbwdaily2 > tr:nth-child(" + (i + 1) + ") > td:nth-child(5) > span:nth-child(1)").InnerHtml;
weatherVO.DirectionWind = doc.QuerySelector("#tbwdaily2 > tr:nth-child(" + (i + 1) + ") > td:nth-child(6) > dl:nth-child(1) > dt:nth-child(1)").InnerHtml;
weatherVO.SpeedWind = doc.QuerySelector("#tbwdaily2 > tr:nth-child(" + (i + 1) + ") > td:nth-child(6) > dl:nth-child(1) > dd:nth-child(2) > span:nth-child(1)").InnerHtml;
weatherVO.Humidity = doc.QuerySelector("#tbwdaily2 > tr:nth-child(" + (i + 1) + ") > td:nth-child(7)").InnerHtml;
weatherVO.TemperatureFeels = doc.QuerySelector("#tbwdaily2 > tr:nth-child(" + (i + 1) + ") > td:nth-child(8) > span:nth-child(1)").InnerHtml;
weather[i] = weatherVO;
}
cityVO.Weather = weather.ToArray();
for (int i = 0; i < 4; i++)
{
weather[i] = null;
}
weather = null;
doc.Dispose();
doc = null;
return(cityVO);
}
/// <summary>
/// Dispose underlying document
/// </summary>
public void Dispose()
{
_document.Dispose();
}
/// <summary>
/// Освободить ресурсы
/// </summary>
public void Dispose()
{
_html?.Dispose();
}
