static public void Execute(atom.Trace context, int level, IHtmlCollection <IElement> data)
{
if ((data != null) && (data.Length > 0))
{
context.
SetComment(GetArraySize(data), "").
Send(NAME.SOURCE.PREVIEW, NAME.TYPE.FOLDER, level, "[[Links]]");
foreach (var a_Context in data)
{
var a_Name1 = GetFirstLine(NodeExtensions.Text(a_Context));
var a_Name2 = GetFirstLine(NodeExtensions.HyperReference(a_Context, a_Context.GetAttribute("Href"))?.Href);
if (string.IsNullOrEmpty(a_Name1))
{
a_Name1 = GetFirstLine(a_Context.InnerHtml);
}
if ((string.IsNullOrEmpty(a_Name1) == false) || (string.IsNullOrEmpty(a_Name2) == false))
{
context.
SetUrl(a_Name2, "").
SetComment("[[Link]]", HINT.DATA_TYPE).
Send(NAME.SOURCE.PREVIEW, NAME.TYPE.INFO, level + 1, a_Name1);
}
}
}
}
private static void StartWalkAroundCategories(IHtmlCollection <IElement> nodes)
{
foreach (var groupNode in nodes.Select(m => m.Children).FirstOrDefault().Select(t => t.Children))
{
WalkAroundCategories(1, groupNode);
}
}
private T CreatePizza <T>(IElement td) where T : Product
{
string weightStr = string.Empty;
string priceStr = string.Empty;
string name = GetData(td, "a.product-link");
string desc = GetData(td, "div.product-description__content");
string fullImage = GetData(td, "img.lazy", "data-src");
IHtmlCollection <IElement> a = td.QuerySelectorAll("label.variant-pr");
foreach (var iElement in a)
{
priceStr = iElement.QuerySelector("span.woocommerce-Price-amount.amount").Text();
var size1 = iElement.QuerySelector("span.radio-custom").Text();
}
string countStr = SearchCount2(desc);
//string weightStr = GetData(td, "span.product-checksize");
int? weight = StringToInt(weightStr);
int? price = StringToInt(priceStr);
int? count = StringToInt(countStr);
string logo = GetLogoPath(RestaurantType.RollClub.ToString());
T product = CreatProduct <T>(name, desc, weight, count, price, fullImage, ((int)RestaurantType.RollClub).ToString(), logo);
return(product);
}
public async Task <ICollection <NewProductModel> > WatchSite()
{
ICollection <NewProductModel> result = new List <NewProductModel>();
// Rise flag that StartWatch method is running.
_isActive = true;
try
{
// Setup the configuration to support document loading. Add css selectors support.
IConfiguration config = Configuration.Default.WithDefaultLoader().WithCss();
// Asynchronously get the document in a new context using the configuration
IDocument document = await BrowsingContext.New(config).OpenAsync(address);
// Get all products link elements
IHtmlCollection <IElement> productPageLinkElements = document.QuerySelectorAll("a.product_name");
result = await GetProductsDescriptions(productPageLinkElements, address, config);
}
catch (Exception)
{
// Ensure method activity flag is set to false.
_isActive = false;
throw;
}
_isActive = false;
return(result);
}
internal static void ReplaceNodeTags(IHtmlCollection <IElement> nodeList, string newTagName)
{
foreach (var node in nodeList)
{
SetNodeTag(node, newTagName);
}
}
public async Task <ICollection <Deck> > GetDecksAsync(int eventId)
{
string address = $"https://{BaseUrl}/event?e={eventId}&f=ST";
IDocument document = await _context.OpenAsync(address).ConfigureAwait(false);
const string cellSelector = "div.chosen_tr, div.hover_tr";
IHtmlCollection <IElement> cells = document.QuerySelectorAll(cellSelector);
var decks = new List <Deck>();
foreach (IElement cell in cells.Take(8))
{
IElement link = cell.QuerySelector("div:nth-child(2) > a");
if (link != null)
{
string placement = cell.QuerySelector("div:nth-child(1)")?.TextContent ?? string.Empty;
int id = int.Parse(HttpUtility.ParseQueryString(new Uri($"https://{BaseUrl}/{link.GetAttribute("href")}").Query)["d"], CultureInfo.InvariantCulture);
string name = link.TextContent;
string user = cell.QuerySelector("div:nth-child(3)").TextContent;
ICollection <DeckCard> cards = await GetDeckAsync(eventId, id).ConfigureAwait(false);
decks.Add(new Deck
{
Cards = cards,
Id = id,
TournamentId = eventId,
Name = name,
Placement = placement,
User = user,
});
}
}
return(decks);
}
static int DownloadAndFindNext(string url, int page)
{
Console.WriteLine("Downloading page " + page);
const string NextPageSelector = "div.row > div > div.card > div.card-body > div.paginator-top > ul > li.page-item > a[rel=\"next\"]";
//Download source
string path = Path.GetFullPath(@"work\page" + page + ".html");
if (!File.Exists(path) || OverwritePages)
{
using (WebClient client = new WebClient())
{
client.DownloadFile(url, path);
}
}
string source = File.ReadAllText(path);
AngleSharp.Html.Dom.IHtmlDocument document = Parser.ParseDocument(source);
IHtmlCollection <IElement> pages = document.QuerySelectorAll(NextPageSelector);
if (pages.Length != 0)
{
AngleSharp.Html.Dom.IHtmlAnchorElement thisPage = (AngleSharp.Html.Dom.IHtmlAnchorElement)pages[0];
return(DownloadAndFindNext(thisPage.Href, page + 1));
}
else
{
return(page);
}
}
public List <string> GetPageList(IHtmlCollection <IElement> htmlCollection)
{
List <string> pagelist = new List <string>();
//string Url = "";
if (htmlCollection != null)
{
foreach (var divInfo in htmlCollection)
{
var bb = divInfo.QuerySelectorAll("a").Where(i => i.GetAttribute("title").Contains("最末页")).ToList().Last();
var page = bb.GetAttribute("href");
string[] pagestr = page.Split('/');
string pageuri = "";
for (int i = 0; i < pagestr.Length - 1; i++)
{
pageuri = pageuri + pagestr[i] + "/";
}
int pagecount = Convert.ToInt32(pagestr[pagestr.Length - 1]);
for (int i = 1; i <= pagecount; i++)
{
pagelist.Add(pageuri + i);
}
}
}
return(pagelist);
}
/// <summary>
/// Method for gets volume information for manga
/// </summary>
/// <param name="information">Html manga information representation</param>
/// <param name="parsedManga">Object manga representation</param>
private void GetVolume(IHtmlCollection <IElement> information, IManga parsedManga)
{
_mangaVolumeRegex.Match(information[0].TextContent.EmptyStringIfNull())
.GetGroupValue(1, out var textVolumes);
parsedManga.Volumes = textVolumes;
}
private static string GetProjectName(IHtmlCollection <IElement> title)
{
return(title[0]
.InnerHtml
.Split('>')[1]
.Trim());
}
private static string GetClubName(IHtmlCollection <IElement> title)
{
return(title[1]
.InnerHtml
.Substring(0, title[1].InnerHtml.LastIndexOf('-'))
.Trim());
}
public static void ForEach(this IHtmlCollection <IElement> elements, Action <IElement> action)
{
foreach (var element in elements)
{
action(element);
}
}
private void AddProducts(IHtmlCollection <IElement> divsWithProduct, string url)
{
foreach (var div in divsWithProduct)
{
AddProduct(div, url);
}
}
protected virtual string BuildArticleText(IHtmlCollection <IElement> paragraphs)
{
StringBuilder description = new StringBuilder();
foreach (var p in paragraphs)
{
if (p.TagName.ToLower().StartsWith("h"))
{
description.AppendLine($"<h4>{p.TextContent.Trim()}</h4>");
}
else
{
// Watch for the older style line breaks and convert to proper paragraphs
if (p.InnerHtml.Contains("<br>"))
{
Log.Information("Replacing old style line breaks with paragraph tags");
string value = p.InnerHtml.Replace("<br>", "</p><p>");
description.AppendLine($"<p>{value}</p>");
}
else
{
description.AppendLine($"<p>{p.InnerHtml}</p>");
}
}
}
return(description.ToString());
}
/// <summary>
/// Scrapes a document for schedule
/// </summary>
/// <param name="scheduleDocument">Schedule document to be scraped</param>
/// <returns>Scraped ScheduleModel</returns>
public async Task <Schedule.Week> ScrapeSchedule(IDocument scheduleDocument)
{
IHtmlCollection <IElement> matchupContainers = scheduleDocument.HtmlDocument.QuerySelectorAll(".schedules-list .schedules-table");
Schedule.Week weekModel = new Schedule.Week();
foreach (var matchupContainer in matchupContainers)
{
Regex regex = new Regex("(?:<!--).*(-->)");
MatchCollection results = regex.Matches(matchupContainer.InnerHtml.ToString());
foreach (var result in results)
{
weekModel.RawElements.Add(result);
Console.WriteLine(result.ToString());
}
}
weekModel.ScheduleDays = GetGameDays(weekModel.RawElements);
foreach (Day scheduleDay in weekModel.ScheduleDays)
{
scheduleDay.Games = GetGames(scheduleDay);
}
return(weekModel);
}
private List <LeagueModel> processLeagues(IDocument response)
{
var leagues = new LeagueModels();
IDocument document = response;
IHtmlCollection <IElement> leagueListTable = document.QuerySelectorAll("table tr:not(.LSport)");
foreach (IElement row in leagueListTable)
{
var league = new LeagueModel();
var cols = row.QuerySelectorAll("td");
cols.Each((td, index) => {
var href = "";
var anchor = td.FirstElementChild;
if (anchor != null)
{
href = anchor.GetAttribute("href");
}
switch (index)
{
case 0: league.Title = td.TextContent; break;
case 1: league.Fixture = href; break;
case 2: league.Standing = href; break;
}
});
leagues.Add(league);
}
return(leagues);
}
//Mark Hale
//Checks if employees details correctly
public async Task GET_DetailsDisplaysCorrectEmployeeInformation()
{
// Arrange
// Create variables to represent data to be tested
string url = "/employee/Details/2";
string firstName = "Taylor";
string lastName = "Gulley";
string dept = "Marketing";
string fullName = firstName + " " + lastName;
string computerManufacturer = "Schmicrosoft";
string computerMake = "Schmurface Pro";
string wholeComputer = computerManufacturer + " " + computerMake;
string firstTrainingProgramName = "POS Training";
// Act
// Get HTTP response from variable defined above
HttpResponseMessage response = await _client.GetAsync(url);
// Assert
// Check if there is any data is displayed on details/1
response.EnsureSuccessStatusCode(); // Status Code 200-299
Assert.Equal("text/html; charset=utf-8",
response.Content.Headers.ContentType.ToString());
//Check if data displayed matches data in database
IHtmlDocument indexPage = await HtmlHelpers.GetDocumentAsync(response);
IHtmlCollection <IElement> dds = indexPage.QuerySelectorAll("dd");
Assert.Contains(dds, dd => dd.TextContent.Trim() == fullName);
Assert.Contains(dds, dd => dd.TextContent.Trim() == dept);
Assert.Contains(dds, dd => dd.TextContent.Trim() == wholeComputer);
Assert.Contains(dds, dd => dd.TextContent.Trim() == firstTrainingProgramName);
}
private void AddVocalPlay(IHtmlCollection <IElement> line, int it, ref Movie movie)
{
string singingActor = string.Empty, voiceActor = string.Empty, singingDubber = string.Empty, voiceDubber = string.Empty;
var character = ExtractCastInfo(line[0].TextContent);
ExtractActor(ExtractCastInfo(line[1].TextContent), ref singingActor, ref voiceActor);
ExtractActor(ExtractCastInfo(line[it].TextContent), ref singingDubber, ref voiceDubber);
movie.cast.Add(
new CastMember()
{
character = character + " (Voce)",
actor = voiceActor,
dubber = new DubberRef()
{
name = voiceDubber
}
}
);
movie.cast.Add(
new CastMember()
{
character = character + " (Canto)",
actor = singingActor,
dubber = new DubberRef()
{
name = singingDubber
}
}
);
}
static public void Execute(atom.Trace context, int level, IHtmlCollection <IHtmlImageElement> data)
{
if ((data != null) && (data.Length > 0))
{
context.
SetComment(GetArraySize(data), "").
Send(NAME.SOURCE.PREVIEW, NAME.TYPE.FOLDER, level, "[[Images]]");
foreach (var a_Context in data)
{
if (string.IsNullOrEmpty(a_Context.Source) == false)
{
context.
SetUrl(a_Context.Source, "").
SetComment("[[Image]]", HINT.DATA_TYPE).
Send(NAME.SOURCE.PREVIEW, NAME.TYPE.INFO, level + 1, GetFileName(a_Context.Source));
{
context.Send(NAME.SOURCE.PREVIEW, NAME.TYPE.PREVIEW, level + 2);
context.Send(NAME.SOURCE.PREVIEW, NAME.TYPE.PREVIEW, level + 2);
context.Send(NAME.SOURCE.PREVIEW, NAME.TYPE.PREVIEW, level + 2);
context.Send(NAME.SOURCE.PREVIEW, NAME.TYPE.PREVIEW, level + 2);
}
}
}
}
}
public IEnumerable <AnimeInfo> ToAnimeInfo(IHtmlCollection <IElement> animeElements)
{
List <AnimeInfo> animeInfos = new List <AnimeInfo>();
foreach (var elem in animeElements.Select((s, i) => new { element = s, index = i }))
{
var animeInfo = new AnimeInfo();
animeInfo.Id = elem.index;
//タイトル取得
//div.mTitle>h2>a>text
animeInfo.Title = GetTitle(elem.element);
//作品公式URL取得
//a.officialSite
animeInfo.OfficialUrl = GetOfficialUrl(elem.element);
//放送局,放送開始日付,放送時間 取得
//div.schedule
//2,3番目のtr内にそれぞれ3つずつtdがある
//放送局
//td内の1つ目のspan.station
//初回放送開始日時
//td内の2つ目のspan
//span.stationをすべて取得して,それとその次の要素で取得できそう
animeInfo.Schedules = GetSchedules(elem.element);
animeInfos.Add(animeInfo);
}
return(animeInfos);
}
public object ExtractProperties(CrawlResult crawlResult, IHtmlCollection <IElement> elements)
{
Directory.CreateDirectory(@"c:\temp\WebScraper");
var result = new List <object>();
using (var client = new HttpClient())
{
foreach (var element in elements.Where(e => e.TagName.Equals("a", StringComparison.OrdinalIgnoreCase)))
{
var href = new Uri(crawlResult.RequestUrl, new Uri(element.Attributes["href"].Value, UriKind.RelativeOrAbsolute));
var fileName = Path.Combine(@"c:\temp\WebScraper", Path.GetFileName(href.LocalPath));
if (File.Exists(fileName) == false)
{
Console.WriteLine($"Downloading {href} to {fileName}");
using (var f = File.OpenWrite(fileName))
using (var s = client.GetStreamAsync(href).Result)
{
s.CopyTo(f);
}
}
else
{
Console.WriteLine($"Skipping download of {href} to {fileName}");
}
result.Add(new { FileName = fileName, Title = element.TextContent });
}
return(result);
}
}
public object ExtractProperties(CrawlResult crawlResult, IHtmlCollection <IElement> elements)
{
Directory.CreateDirectory(@"c:\temp\WebScraper");
using (var client = new HttpClient())
{
foreach (var img in elements.SelectMany(e => e.QuerySelectorAll("img")))
{
var src = new Uri(crawlResult.RequestUrl, new Uri(img.Attributes["src"].Value, UriKind.RelativeOrAbsolute));
var fileName = Path.Combine(@"c:\temp\WebScraper", Path.GetFileName(src.LocalPath));
if (File.Exists(fileName) == false)
{
Console.WriteLine($"Downloading {src} to {fileName}");
using (var f = File.OpenWrite(fileName))
using (var s = client.GetStreamAsync(src).Result)
{
s.CopyTo(f);
}
}
else
{
Console.WriteLine($"Skipping download of {src} to {fileName}");
}
img.SetAttribute("data-local-src", fileName);
}
}
return(elements.Select(this.htmlSelector).Aggregate((prod, next) => prod + "\n" + next));
}
public async Task <IEnumerable <(int id, string name, int rating, DateTime date)> > GetTournamentsAsync()
{
string address = $"https://{BaseUrl}/format?f=ST";
IDocument document = await _context.OpenAsync(address).ConfigureAwait(false);
const string eventsSelector =
".page > div:nth-child(1) > table:nth-child(1) > tbody:nth-child(1) > tr:nth-child(1) > td:nth-child(2) > table:nth-child(3) > tbody:nth-child(1) > tr:not(:first-child)," +
".page > div:nth-child(1) > table:nth-child(1) > tbody:nth-child(1) > tr:nth-child(1) > td:nth-child(2) > table:nth-child(1) > tbody:nth-child(1) > tr:not(:first-child)";
IHtmlCollection <IElement> eventsCells = document.QuerySelectorAll(eventsSelector);
var events = new List <(int id, string name, int rating, DateTime date)>();
foreach (IElement cell in eventsCells)
{
IElement link = cell.QuerySelector("td:nth-child(1) > a");
int id = int.Parse(HttpUtility.ParseQueryString(new Uri($"https://{BaseUrl}/{link.GetAttribute("href")}").Query)["e"], CultureInfo.InvariantCulture);
string name = link.TextContent;
int rating = cell.QuerySelector("td:nth-child(2)").ChildElementCount;
if (cell.QuerySelector("td:nth-child(2)").FirstElementChild.GetAttribute("src") == "graph/bigstar.png")
{
rating = 4;
}
DateTime date = DateTime.ParseExact(cell.QuerySelector("td:nth-child(3)").TextContent, "dd/MM/yy", CultureInfo.CurrentCulture);
if (!events.Select(x => x.id).Contains(id))
{
events.Add((id, name, rating, date));
}
}
return(events);
}
public IEnumerable <SubtitleModel> Parse(string xml)
{
var parser = new HtmlParser();
var document = parser.Parse(xml);
IHtmlCollection <IElement> elements = document.QuerySelectorAll("body div[region=\"subtitleArea\"] > p");
List <SubtitleModel> result = new List <SubtitleModel>();
foreach (var element in elements)
{
var begin = element.Attributes.FirstOrDefault(x => string.Equals(x.Name, "begin", StringComparison.OrdinalIgnoreCase))?.Value?.Trim();
var end = element.Attributes.FirstOrDefault(x => string.Equals(x.Name, "end", StringComparison.OrdinalIgnoreCase))?.Value?.Trim();
var value = element.TextContent?.Trim();
if (string.IsNullOrEmpty(begin)
||
string.IsNullOrWhiteSpace(end)
||
string.IsNullOrWhiteSpace(value)
)
{
continue;
}
result.Add(new SubtitleModel
{
Start = ParseTime(begin),
End = ParseTime(end),
Value = value,
});
}
return(result);
}
private static void WalkAroundCategories(int deep, IHtmlCollection <IElement> level)
{
foreach (var child in level)
{
var category = ((IHtmlAnchorElement)child.QuerySelector(Resource.TIUCategoryNameClass));
if (category != null && category is IHtmlAnchorElement)
{
var t = MakeRequestAsync(category.Href, Resource.TIUCategoryClass);
t.Wait();
var innerLevel = t.Result.Select(m => m.Children).FirstOrDefault();
if (innerLevel != null)
{
Categories.Add(new Category {
ProductUrl = category.Href, Description = category.TextContent.Trim()
});
WalkAroundCategories(deep + 1, innerLevel);
}
if (innerLevel == null)
{
Categories.Add(new Category {
ProductUrl = category.Href, Description = category.TextContent.Trim()
});
ScanPageWithProducts(category.Href, category.TextContent.Trim());
}
}
}
}
protected override void Parse(IDocument document)
{
try
{
name = new string[]
{
document.GetElementsByClassName("line-1")[0].TextContent.Replace("смотреть онлайн", ""),
document.GetElementsByClassName("line-2")[0].TextContent
};
IHtmlCollection <IElement> descDoc = document.GetElementsByClassName("card m-description-item");
bool flag = descDoc.Length > 0;
if (flag)
{
description = descDoc[0].TextContent.Replace("Описание аниме", "");
}
imageUrl = "https://smotret-anime.ru" + document.QuerySelector("img[itemprop='contentUrl']").GetAttribute("src");
count = GetCountVideo(document.Body);
}
catch (Exception er)
{
WriteLog(string.Concat(new object[]
{
er.Source,
":",
er.TargetSite,
":",
er.Message
}));
}
}
// This method gets element with product description and extracts all text from it.
// Method is hardcoded to parse description from exact site. But all this service is devoted to explore same exact site.
private string GetTextFromDomElement(IElement elementsBlock)
{
if (elementsBlock == null)
{
return(null);
}
StringBuilder stringBuilder = new StringBuilder();
IHtmlCollection <IElement> elements = elementsBlock.QuerySelectorAll("*");
if (elements != null)
{
foreach (var element in elements)
{
if (element.TagName.ToLower() == "p")
{
if (!string.IsNullOrEmpty(element.TextContent))
{
stringBuilder.AppendLine(element.TextContent);
}
}
else if (element.TagName.ToLower() == "ul")
{
ParseUlElement(element, stringBuilder);
}
else if (element.TagName.ToLower() == "br")
{
stringBuilder.AppendLine();
}
}
}
string result = stringBuilder.ToString();
stringBuilder.Clear();
return(result);
}
// David Taylor
// Department Details displays all employees for department
public async Task Get_DetailsDisplayEmployees()
{
// Arrange
// Creates variables to represent data to be tested
Employee employee = (await GetAllEmployees()).First();
string url = $"/department/details/1";
string employeeFirstName = employee.FirstName;
string employeeLastName = employee.LastName;
// Act
// Gets HTTP response for data represented above
HttpResponseMessage response = await _client.GetAsync(url);
// Assert
// Checks if there is any data represented on details
response.EnsureSuccessStatusCode(); // Status Code 200-299
Assert.Equal("text/html; charset=utf-8",
response.Content.Headers.ContentType.ToString());
// Checks if data displayed represents data in database
IHtmlDocument detailPage = await HtmlHelpers.GetDocumentAsync(response);
IHtmlCollection <IElement> viewData = detailPage.QuerySelectorAll("h2");
Assert.Contains(viewData, h2 => h2.TextContent.Trim() == "Marketing");
IHtmlCollection <IElement> lis = detailPage.QuerySelectorAll("li");
Assert.Contains(lis, li => li.TextContent.Trim() == employeeFirstName + " " + employeeLastName);
}
/// <summary>
/// <para>Iterate over a NodeList, which doesn't natively fully implement the Array
/// interface.</para>
/// <para>For convenience, the current object context is applied to the provided
/// iterate function.</para>
/// </summary>
/// <param name="nodeList">The nodes to operate on</param>
/// <param name="fn">The iterate function</param>
/// <return>void</return>
internal static void ForEachElement(IHtmlCollection <IElement> nodeList, Action <IElement> fn)
{
for (int a = 0; a < nodeList.Length; a++)
{
fn(nodeList[a]);
}
}
static public void Execute(atom.Trace context, int level, IHtmlCollection <IHtmlAnchorElement> data)
{
if ((data != null) && (data.Length > 0))
{
context.
SetComment(GetArraySize(data), "").
Send(NAME.SOURCE.PREVIEW, NAME.TYPE.FOLDER, level, "[[Anchors]]");
foreach (var a_Context in data)
{
var a_Name = GetFirstLine(a_Context.GetAttribute("Name"));
if (string.IsNullOrEmpty(a_Name) == false)
{
context.
SetComment("[[Anchor]]", HINT.DATA_TYPE).
Send(NAME.SOURCE.PREVIEW, NAME.TYPE.INFO, level + 1, a_Name);
{
Send(context, NAME.TYPE.VARIABLE, level + 2, "[[Download]]", a_Context.Download);
Send(context, NAME.TYPE.VARIABLE, level + 2, "[[Language]]", a_Context.TargetLanguage);
Send(context, NAME.TYPE.VARIABLE, level + 2, "[[Relation]]", a_Context.Relation);
Send(context, NAME.TYPE.VARIABLE, level + 2, "[[Target]]", a_Context.Target);
Send(context, NAME.TYPE.VARIABLE, level + 2, "[[Type]]", a_Context.Type);
}
}
}
}
}
private static IResultDetail ParseDetail(IHtmlCollection cells)
{
var type = ParseResultType(cells[DetailTypeIndex].Text);
return new ResultDetail(type,
cells[DetailNameIndex].Text,
cells[DetailDescriptionIndex].Text);
}
private static bool IsSummaryRow(IHtmlElement row, IHtmlCollection cells)
{
return row.Attributes.Count == 0
&& cells.Count == NumSummaryCells
&& HasSummaryCellAttributes(cells.First);
}
private static bool IsDetailRow(IHtmlElement row, IHtmlCollection cells)
{
return HasDetailRowClass(row)
&& cells.Count == NumDetailCells
&& cells.First.Attributes.Count == 0;
}
/// <summary>
/// Parses the apartment list items and calls the scraper for each apartment
/// </summary>
/// <param name="items">List of html LI items</param>
/// <returns>List of tasks that return the scraped Apartments</returns>
private List<Task<ApartmentDTO>> ScrapApartmentList(IHtmlCollection<IElement> items)
{
var currentNeighborhood = string.Empty;
var apartmentTaskList = new List<Task<ApartmentDTO>>();
foreach (var item in items)
{
var role = item.Attributes["data-role"];
if (role != null && role.Value == "list-divider")
{
currentNeighborhood = item.TextContent;
}
else
{
var itemLink = item.QuerySelector("a");
var href = itemLink.Attributes["href"];
if (href != null)
{
var apartmentUrl = href.Value;
var titleElementHtml = item.QuerySelector("h2").InnerHtml;
var titleMatch = Regex.Match(titleElementHtml, @"(.*)<br>(\d+) Berlin");
var task = Scraper.ScrapUrlAsync("https://akelius.de" + apartmentUrl)
.ContinueWith(aptTask =>
{
var apartment = aptTask.Result;
apartment.Street = titleMatch.Groups[1].Value;
apartment.ZipCode = titleMatch.Groups[2].Value;
return apartment;
});
apartmentTaskList.Add(task);
}
}
}
return apartmentTaskList;
}
