private static async Task SeedAirCoolers(ApplicationDbContext dbContext) { using (SqlConnection connection = new SqlConnection(Config.ConnectionString)) { connection.Open(); var command = new SqlCommand(SqlCommands.SelectAirCoolersSql, connection); var airCoolers = new List <AirCooler>(); SqlDataReader reader = await command.ExecuteReaderAsync(); try { while (reader.Read()) { var airCooler = new AirCooler { Brand = reader["Brand"].ToString(), CPUSocketCompatibility = reader["CPUSocketCompatibility"].ToString(), Image = (byte[])reader["Image"], ImgUrl = reader["ImgUrl"].ToString(), FanSize = reader["FanSize"].ToString(), HeatsinkDimensions = reader["HeatsinkDimensions"].ToString(), Name = reader["Name"].ToString(), MaxCPUCoolerHeight = reader["MaxCPUCoolerHeight"].ToString(), PowerConnector = reader["PowerConnector"].ToString(), RPM = reader["RPM"].ToString(), Weight = reader["Weight"].ToString(), CategoryId = 6, }; airCoolers.Add(airCooler); } dbContext.AirCoolers.AddRange(airCoolers); await dbContext.SaveChangesAsync(); } catch (Exception) { throw; } finally { reader.Close(); } } }
public async Task <IEnumerable <AirCooler> > GatherAirCoolerData() { var videoCards = new List <AirCooler>(); var productUrls = new List <string>(); var parser = new HtmlParser(); var client = new HttpClient(); for (int page = 1; page <= 96; page++) { Console.Write($"{page} => "); var url = $"https://www.newegg.com/Desktop-Memory/SubCategory/ID-574/Page-{page}"; string htmlContent = null; for (var i = 0; i < 10; i++) { try { var response = await client.GetAsync(url); htmlContent = await response.Content.ReadAsStringAsync(); break; } catch { Console.Write('!'); Thread.Sleep(500); } } if (string.IsNullOrWhiteSpace(htmlContent)) { break; } var document = await parser.ParseDocumentAsync(htmlContent); var elements = document.GetElementsByClassName("item-container "); if (elements.Length == 0) { break; } foreach (var element in elements) { string pcPartPickerUrl = null; var options = element.InnerHtml.Split('\n', StringSplitOptions.RemoveEmptyEntries); foreach (var option in options) { if (option.Contains("href=") && option.Contains("item-img")) { var productUrlUntrimmed = option.Substring(option.IndexOf('/')); var productUrl = productUrlUntrimmed.Substring(0, productUrlUntrimmed.Length - 19); pcPartPickerUrl = "https:" + productUrl; productUrls.Add(pcPartPickerUrl); } } } } int count = 0; foreach (var url in productUrls) { string htmlContent = null; for (var i = 0; i < 10; i++) { try { var response = await client.GetAsync(url); htmlContent = await response.Content.ReadAsStringAsync(); break; } catch { Console.Write('!'); Thread.Sleep(500); } } Console.WriteLine(count); count++; var document = await parser.ParseDocumentAsync(htmlContent); var manufacturerInfo = document.GetElementById("MfrContact"); string productUrl = string.Empty; if (manufacturerInfo != null) { productUrl = manufacturerInfo.GetElementsByTagName("a")[0].InnerHtml; } var productSpecs = document.GetElementById("detailSpecContent"); string productSpecsInnerHtml = string.Empty; if (productSpecs == null) { continue; } productSpecsInnerHtml = productSpecs.InnerHtml; var specs = productSpecsInnerHtml.Split("<dl>", StringSplitOptions.RemoveEmptyEntries); var productName = string.Empty; if (specs[0].Contains("<span>")) { productName = specs[0].Substring(specs[0].IndexOf("<span>") + 6).Trim(); productName = productName.Substring(0, productName.IndexOf("</span>")).Trim(); } var videoCard = new AirCooler { Name = productName, ProductUrl = productUrl, }; var imgHtmlElemnts = document.GetElementsByName("gallery"); string imgHtml = string.Empty; if (imgHtmlElemnts.Length > 0) { imgHtml = imgHtmlElemnts[0].InnerHtml; imgHtml = imgHtml.Substring(imgHtml.IndexOf("//") + 2); imgHtml = imgHtml.Substring(0, imgHtml.IndexOf(">") - 1); } if (imgHtml.Length > 0) { videoCard.ImgUrl = imgHtml; } foreach (var spec in specs) { if (spec.Contains("<dt>") && spec.Contains("<dd>")) { var replaced = spec.Replace("<dt>", "|"); replaced = replaced.Replace("</dt>", "|"); replaced = replaced.Replace("<dd>", "|"); replaced = replaced.Replace("</dd>", "|"); var specsList = replaced.Split('|', StringSplitOptions.RemoveEmptyEntries); var specName = specsList[0]; var specValue = specsList[1]; if (specName.Contains("a data")) { specName = specName.Substring(specName.IndexOf(">") + 1); specName = specName.Substring(0, specName.IndexOf("<")); } switch (specName) { case "Brand": videoCard.Brand = specValue; break; case "Fan Size": videoCard.FanSize = specValue; break; case "CPU Socket Compatibility": videoCard.CPUSocketCompatibility = specValue; break; case "RPM": videoCard.RPM = specValue; break; case "Power Connector": videoCard.PowerConnector = specValue; break; case "Max CPU Cooler Height": videoCard.MaxCPUCoolerHeight = specValue; break; case "Heatsink Dimensions": videoCard.HeatsinkDimensions = specValue; break; case "Weight": videoCard.Weight = specValue; break; default: break; } } } videoCards.Add(videoCard); } return(videoCards); }
private static void AddImagesToThePartsCpuToHdd() { using (var context = new ApplicationDbContext()) { var cpus = context.Cpus .Where(x => x.Name != " ").ToList(); context.SaveChanges(); var cpusToAdd = new List <Cpu>(); Console.WriteLine(cpus.Count); foreach (var cpu in cpus) { Console.WriteLine(cpu.Id); var imgArray = TransformImage(cpu.ImgUrl).GetAwaiter().GetResult(); var cpuToAdd = new Cpu { Brand = cpu.Brand, CPUSocketType = cpu.CPUSocketType, Image = imgArray, ImgUrl = cpu.ImgUrl, ManufacturingTech = cpu.ManufacturingTech, Model = cpu.Model, Name = cpu.Name, NumberOfCores = cpu.NumberOfCores, NumberOfThreads = cpu.NumberOfThreads, ProcesorType = cpu.ProcesorType, TDP = cpu.TDP, Series = cpu.Series, ProductUrl = cpu.ProductUrl, }; cpusToAdd.Add(cpuToAdd); } context.Cpus.RemoveRange(context.Cpus); context.Cpus.AddRange(cpusToAdd); context.SaveChanges(); } using (var context = new ApplicationDbContext()) { var airCoolers = context.AirCoolers.Where(x => x.Name != " ").ToList(); context.SaveChanges(); var airCoolersToAdd = new List <AirCooler>(); Console.WriteLine(airCoolers.Count); foreach (var airCooler in airCoolers) { Console.WriteLine(airCooler.Id); var imgArray = TransformImage(airCooler.ImgUrl).GetAwaiter().GetResult(); var airCoolerToAdd = new AirCooler { Brand = airCooler.Brand, FanSize = airCooler.FanSize, CPUSocketCompatibility = airCooler.CPUSocketCompatibility, HeatsinkDimensions = airCooler.HeatsinkDimensions, Image = imgArray, ImgUrl = airCooler.ImgUrl, MaxCPUCoolerHeight = airCooler.MaxCPUCoolerHeight, Name = airCooler.Name, PowerConnector = airCooler.PowerConnector, RPM = airCooler.RPM, Weight = airCooler.Weight, ProductUrl = airCooler.ProductUrl, }; airCoolersToAdd.Add(airCoolerToAdd); } context.AirCoolers.RemoveRange(context.AirCoolers); context.AirCoolers.AddRange(airCoolersToAdd); context.SaveChanges(); } using (var context = new ApplicationDbContext()) { var cases = context.Cases.Where(x => x.Name != " ").ToList(); context.SaveChanges(); var casesToAdd = new List <Case>(); Console.WriteLine(cases.Count); foreach (var @case in cases) { Console.WriteLine(@case.Id); var imgArray = TransformImage(@case.ImgUrl).GetAwaiter().GetResult(); var caseToAdd = new Case { Brand = @case.Brand, MaxCPUCoolerHeightAllowance = @case.MaxCPUCoolerHeightAllowance, MaxGPULengthAllowance = @case.MaxGPULengthAllowance, CaseMaterial = @case.CaseMaterial, Color = @case.Color, ExpansionSlots = @case.ExpansionSlots, Image = imgArray, ImgUrl = @case.ImgUrl, MotherboardCompatibility = @case.MotherboardCompatibility, Name = @case.Name, SidePanelWindow = @case.SidePanelWindow, Type = @case.Type, ProductUrl = @case.ProductUrl, }; casesToAdd.Add(caseToAdd); @case.Image = imgArray; } context.Cases.RemoveRange(context.Cases); context.Cases.AddRange(casesToAdd); context.SaveChanges(); } using (var context = new ApplicationDbContext()) { var hdds = context.HardDiskDrives.Where(x => x.Name != " ").ToList(); context.SaveChanges(); var hardDisksToAdd = new List <HardDiskDrive>(); Console.WriteLine(hdds.Count); foreach (var hdd in hdds) { Console.WriteLine(hdd.Id); var imgArray = TransformImage(hdd.ImgUrl).GetAwaiter().GetResult(); var hardToAdd = new HardDiskDrive { Brand = hdd.Brand, Cache = hdd.Cache, Capacity = hdd.Capacity, FormFactor = hdd.FormFactor, Image = imgArray, ImgUrl = hdd.ImgUrl, Interface = hdd.Interface, Name = hdd.Name, RPM = hdd.RPM, Usage = hdd.Usage, ProductUrl = hdd.ProductUrl, }; hardDisksToAdd.Add(hardToAdd); hdd.Image = imgArray; } context.HardDiskDrives.RemoveRange(context.HardDiskDrives); context.HardDiskDrives.AddRange(hardDisksToAdd); context.SaveChanges(); } }