public Product Add(ProductRepository spiritRepository) { var product = new Product { Name = "yeee" }; var x = spiritRepository.Add(product); return x; }
private async Task Scrape(int Id) { //Debug.WriteLine("scraping: " + Id); HttpResponseMessage r = new HttpResponseMessage(); try { r = await GetPageEntity(id: Id.ToString()); } catch { failedScrapes += 1; failedNetwork += 1; //Debug.WriteLine("[" + Id + "] FAIL: -> HTTP request failed internally?"); return; } if (r.StatusCode != System.Net.HttpStatusCode.OK) { Debug.WriteLine(" <!> != 200"); } if (r.RequestMessage.RequestUri.AbsoluteUri.ToLower().Contains("http://www.thedrinkshop.com/notfound.php")) { Debug.WriteLine(" <!> Not Found"); return; }; //if(Page.Contains("Please confirm you are above the legal drinking age in your country")) { Debug.WriteLine(" <!> Age gate"); return; } //Debug.WriteLine(""); //ew var Page = await r.Content.ReadAsStringAsync(); var packdoc = new HtmlAgilityPack.HtmlDocument(); packdoc.LoadHtml(Page); var doc = packdoc.DocumentNode.Descendants(); var product = new Demo_core.Models.DB.Product(); var tempProductName = doc .Where(a => a.Name == "img") .FirstOrDefault(x => x.Attributes.Any(b => b.OriginalName == "itemprop" && b.Value == "image") && x.Attributes.Any(z => z.OriginalName == "class" && z.Value == "mainImage")) ?.Attributes.FirstOrDefault(y => y.OriginalName == "alt"); if (tempProductName == null) { failedScrapes += 1; //Debug.WriteLine("[" + Id + "] FAIL: -> no name node"); return; } product.Name = tempProductName.Value; //Debug.WriteLine("[" + Id + "] Name: " + product?.Name); if (string.IsNullOrEmpty(product.Name)) { failedScrapes += 1; //Debug.WriteLine("[" + Id + "] FAIL: -> no name"); return; } var tempDescription = doc .Where(a => a.Name == "span") .FirstOrDefault(b => b.Attributes.Any(x => x.OriginalName == "itemprop" && x.Value == "description")); if (tempDescription == null) { failedScrapes += 1; //Debug.WriteLine("[" + Id + "] FAIL: -> no description"); return; } product.Description = Regex.Replace(tempDescription.InnerHtml, @"\<.+\>", string.Empty); //Debug.WriteLine("[" + Id + "] Description: " + product.Description); //<-- var spans = doc .Where(a => a.Name == "span") .FirstOrDefault(x => x.Attributes.Any(b => b.OriginalName == "class" && b.Value == "smallHeaderText")) .SelectNodes("./following-sibling::text()") .Select(g => g.InnerHtml.Replace(" ", "").Replace("\n", "").Replace(";", "")) .Where(y => !Regex.IsMatch(y, @"^[, ]*$") ) .Select(p => p.Trim()).ToList(); //product.Abv = new Abv //{ // Percentage = float.Parse(Regex.Replace(spans[1], @"\D", string.Empty)) //}; //product.Producer = new Producer //{ // Name = spans[2] //}; //product.CountryOfOrigin = new CountryOfOrigin //{ // Name = spans[3] //}; //product.Category = new Category //{ // Name = spans[4] //}; //product.Brand = new Brand //{ // Name = spans[2] //}; var rg = new Regex(@"<[^>]*>"); var bottlePropIni = doc .Where(x => x.Name == "table") .FirstOrDefault(b => b.Attributes.Any(t => t.OriginalName == "class" && t.Value == "priceTable")); if (bottlePropIni == null || bottlePropIni.ChildNodes.Count <=2 ) { failedScrapes += 1; //Debug.WriteLine("[" + Id + "] FAIL: -> no bottles"); return; } var bottleProps = bottlePropIni.ChildNodes.Where(y => y.Name == "tr").ToList()[1] .ChildNodes.Select(u => u.InnerHtml).ToArray().Where((val, x) => (new List<int> {0, 2, 4}).Contains(x)).ToList() .Select(i => rg.Replace(i, "")).ToList(); var tempCentiliters = new float(); if (!float.TryParse(Regex.Match(bottleProps[0], @"^\d*").Value, out tempCentiliters)) { failedScrapes += 1; //Debug.WriteLine("[" + Id + "] FAIL: -> no bottle centiliters"); return; } product.Bottles = new List<Bottle>(); product.Bottles.Add(new Bottle { SizeInCentiLiters = tempCentiliters, RetailName = bottleProps[0], Price = float.Parse(Regex.Match(bottleProps[2], @"\d*$").Value) }); var imageURL = (doc .Where(x => x.Name == "meta").ToArray()[8]).Attributes["content"].Value; product.Image = new Image(); product.Image.ByteArray = await (await GetPageEntity(asset: imageURL)).Content.ReadAsByteArrayAsync(); product.Image.FileExtension = imageURL.Split('.').Last(); var existing = _prodrepo.GetBy(x => x.Name.ToLower() == product.Name.ToLower()).FirstOrDefault(); if (existing == null) { Debug.WriteLine("[" + Id + "] New - Adding to datastore"); _prodrepo.Add(product); } else { Debug.WriteLine("[" + Id + "] Existing - Updating in datastore"); _prodrepo.Update(existing); } //var test = prodrepo.GetAll(); }