public async Task <ScrapeModel> Scrape(int id)
{
var scrapeModel = new ScrapeModel();
var httpClient = new HttpClient();
var html = await httpClient.GetStringAsync(Fsm.Root + Fsm.Destination);
var htmlDocument = new HtmlDocument();
htmlDocument.LoadHtml(html);
var iterator = 0;
var detailedNode = htmlDocument.DocumentNode.SelectNodes("//a").First(node =>
{
if (node.Attributes.Contains("class") && node.Attributes["class"].Value == "hero-portrait-detailed")
{
if (iterator == id)
{
return(true);
}
iterator++;
}
return(false);
});
var innerHtml = new HtmlDocument();
innerHtml.LoadHtml(detailedNode.InnerHtml);
scrapeModel.PictureRoute = innerHtml.DocumentNode.SelectNodes("//img").Single(node => node.Attributes.Contains("class") && node.Attributes["class"].Value == "portrait")
.Attributes["src"].Value;
scrapeModel.Name = innerHtml.DocumentNode.SelectNodes("//span").Single(node =>
node.Attributes.Contains("class") && node.Attributes["class"].Value == "portrait-title").InnerHtml;
var infoSite = await httpClient.GetStringAsync(Fsm.Root + detailedNode.Attributes["href"].Value);
var infoDocument = new HtmlDocument();
infoDocument.LoadHtml(infoSite);
scrapeModel.Description = infoDocument.DocumentNode.SelectNodes("//p").Single(node => node.Attributes.Contains("class") && node.Attributes["class"].Value == "hero-detail-description").InnerHtml;
return(scrapeModel);
}
public void Index_OneItem_ShouldReturnListOfOneScrapeItem()
{
var scrape = new ScrapeModel();
var mockListQuery = new Mock <IGetScrapeListQuery>();
var mockDetailsQuery = new Mock <IGetScrapeDetailQuery>();
mockListQuery.Setup(q => q.Execute()).Returns(new List <ScrapeModel> {
scrape
});
var controller = new ScrapesController(mockListQuery.Object, mockDetailsQuery.Object, null);
var viewResult = (ViewResult)controller.Index();
var items = (List <ScrapeModel>)viewResult.Model;
Assert.Equal(1, items.Count);
Assert.Contains(scrape, items);
}
public async Task <List <ScrapeModel> > Scrape()
{
var scrapeList = new List <ScrapeModel>();
var httpClient = new HttpClient();
var html = await httpClient.GetStringAsync(Fsm.Root + Fsm.Destination);
var htmlDocument = new HtmlDocument();
htmlDocument.LoadHtml(html);
var detailedNodes = htmlDocument.DocumentNode.SelectNodes("//a").Where(node => node.Attributes.Contains("class") && node.Attributes["class"].Value == "hero-portrait-detailed");
foreach (var detailedNode in detailedNodes)
{
var sm = new ScrapeModel();
var innerHtml = new HtmlDocument();
innerHtml.LoadHtml(detailedNode.InnerHtml);
sm.PictureRoute = innerHtml.DocumentNode.SelectNodes("//img").Single(node => node.Attributes.Contains("class") && node.Attributes["class"].Value == "portrait")
.Attributes["src"].Value;
sm.Name = innerHtml.DocumentNode.SelectNodes("//span").Single(node =>
node.Attributes.Contains("class") && node.Attributes["class"].Value == "portrait-title").InnerHtml;
var infoSite = await httpClient.GetStringAsync(Fsm.Root + detailedNode.Attributes["href"].Value);
var infoDocument = new HtmlDocument();
infoDocument.LoadHtml(infoSite);
sm.Description = infoDocument.DocumentNode.SelectNodes("//p").Single(node => node.Attributes.Contains("class") && node.Attributes["class"].Value == "hero-detail-description").InnerHtml;
scrapeList.Add(sm);
}
return(scrapeList);
}
public async Task <bool> GetHtmlAsync(string typ, int förbrukning, string url, int ElområdeId)
{
DbEl db = new DbEl();
var r = new Random();
if (IsExisting(typ, förbrukning, ElområdeId))
{
RemoveContent(typ, förbrukning, ElområdeId);
}
var httpClient = new HttpClient();
var html = await httpClient.GetStringAsync(url);
var htmlDoc = new HtmlDocument();
htmlDoc.LoadHtml(html);
var AllListElements = htmlDoc.DocumentNode.Descendants("li")
.Where(node => node.GetAttributeValue("class", "")
.StartsWith("panel epk-list-item")).ToList();
var Links = htmlDoc.DocumentNode.Descendants("a")
.Where(node => node.GetAttributeValue("href", "")
.StartsWith("/sv/Elpriskollen/Avtalssida/?ellevid="))
.Select(i => i.GetAttributeValue("href", "").Replace("amp;", "")).ToList();
var headers = AllListElements.Select(i => i.Descendants("h3").Select(x => x.InnerText).ToList()).ToList();
var Price = htmlDoc.DocumentNode.Descendants("div")
.Where(node => node.GetAttributeValue("class", "")
.Equals("epk-list-price col-md-2 col-sm-3 ")).Select(i => i.InnerText).ToList();
var Company = htmlDoc.DocumentNode.Descendants("dd")
.Where(node => node.GetAttributeValue("class", "")
.Equals("epk-list-contract-company")).Select(i => i.InnerText).ToList();
var Contract = htmlDoc.DocumentNode.Descendants("dd")
.Where(node => node.GetAttributeValue("class", "")
.Equals("epk-list-contract-type")).Select(i => i.InnerText).ToList();
var list = new List <ScrapeModel>();
for (int i = 0; i < Links.Count; i++)
{
var model = new ScrapeModel();
model.Company = Company[i].Trim();
model.Contract = Contract[i].Trim();
model.Price = Price[i].Trim();
model.ExtraInfo = headers[i][0].Trim();
model.Typ = typ;
model.Förbrukning = förbrukning;
model.ElområdeId = ElområdeId;
model.Rating = 0;
var extraUrl = "https://www.ei.se" + Links[i];
var extraClient = new HttpClient();
var extraHtml = await httpClient.GetStringAsync(extraUrl);
var extraHtmlDoc = new HtmlDocument();
htmlDoc.LoadHtml(extraHtml);
var ExtraAttributes = htmlDoc.DocumentNode.Descendants("td")
.Where(node => node.GetAttributeValue("class", "")
.StartsWith("value")).Select(node => node.InnerText.Trim()).ToList();
var PaymentMethod = htmlDoc.DocumentNode.Descendants("td")
.Where(node => node.GetAttributeValue("class", "")
.Equals("well")).Select(x => x.InnerText).ToList()[5].Split(',');
model.Autogiro = PaymentMethod.Any(p => p.Trim().Equals("Autogiro")) ? true : false;
model.EFaktura = PaymentMethod.Any(p => p.Trim().Equals("E-faktura")) ? true : false;
model.Pappersfaktura = PaymentMethod.Any(p => p.Trim().Equals("Traditionell")) ? true : false;
var SunSource = htmlDoc.DocumentNode.Descendants("li")
.Where(node => node.GetAttributeValue("class", "")
.Equals("epk-energy-type epk-energy-solar")).Select(x => x.InnerText).ToList();
model.Sol = SunSource.Count > 0 ? true : false;
var WindSource = htmlDoc.DocumentNode.Descendants("li")
.Where(node => node.GetAttributeValue("class", "")
.Equals("epk-energy-type epk-energy-wind")).Select(x => x.InnerText).ToList();
model.Vind = WindSource.Count > 0 ? true : false;
var WaterSource = htmlDoc.DocumentNode.Descendants("li")
.Where(node => node.GetAttributeValue("class", "")
.Equals("epk-energy-type epk-energy-water")).Select(x => x.InnerText).ToList();
model.Vatten = WaterSource.Count > 0 ? true : false;
var BioSource = htmlDoc.DocumentNode.Descendants("li")
.Where(node => node.GetAttributeValue("class", "")
.Equals("epk-energy-type epk-energy-bio")).Select(x => x.InnerText).ToList();
model.Bio = BioSource.Count > 0 ? true : false;
var EnvironmentalSource = htmlDoc.DocumentNode.Descendants("li")
.Where(node => node.GetAttributeValue("class", "")
.Equals("epk-energy-type epk-bramiljoval")).Select(x => x.InnerText).ToList();
model.Miljömärkt = EnvironmentalSource.Count > 0 ? true : false;
model.ÅrsAvgift = ExtraAttributes[0];
model.Engångsavgift = ExtraAttributes[1];
model.Fastpris = ExtraAttributes[2];
model.RörligtInköpsPris = ExtraAttributes[3];
model.RörligtPåslag = ExtraAttributes[4];
model.Miljöpåslag = ExtraAttributes[5];
model.RörligtMiljöpåslag = ExtraAttributes[6];
model.Rabatt = ExtraAttributes[7];
model.Moms = ExtraAttributes[10];
model.Uppsägningstid = ExtraAttributes[15];
model.Automatiskförlängning = ExtraAttributes[16];
model.Omteckningsrätt = ExtraAttributes[17];
list.Add(model);
}
db.ScrapeModels.AddRange(list);
db.SaveChanges();
return(true);
}