public static string GetMemorySpeed(this MemorySpeed me) { switch (me) { case MemorySpeed.Fast: return("1400MHz"); case MemorySpeed.Turbo: return("1600MHz"); default: return("1400MHz"); } }
public async Task <string> ScrapeFromProductPageAsync(string productUrl) { if (productUrl.Contains("Combo")) { var message = "Invalid Product."; this.logger.LogWarning(message); return(message); } var document = await this.Context.OpenAsync(productUrl); var cpuData = this.GetAllTablesRows(document); var integratedGrapicsData = new Dictionary <string, string>(); var cpu = new CPU { Price = this.GetPrice(document), ImageUrl = this.GetImageUrl(document), }; this.logger.LogInformation(productUrl); foreach (var tableRow in cpuData) { // this.logger.LogInformation(tr.InnerHtml); var rowName = tableRow.FirstChild.TextContent.Trim().Replace("<!-- --> ", string.Empty); var rowValue = tableRow.LastElementChild.InnerHtml.Replace("<br><br>", "{n}").Replace("<br>", "{n}").Trim(); switch (rowName) { case "Name": if (this.cpuRepo.AllAsNoTracking().Any(x => x.Name == rowValue)) { var message = "Already exists."; this.logger.LogWarning(message); return(message); } cpu.Name = rowValue; break; case "Brand": var brand = this.brandRepo.All().FirstOrDefault(x => x.Name == rowValue); if (brand == null) { brand = new Brand { Name = rowValue, }; } cpu.Brand = brand; break; case "Model": if (this.cpuRepo.AllAsNoTracking().Any(x => x.Model == rowValue)) { var message = "Already exists."; this.logger.LogWarning(message); return(message); } cpu.Model = rowValue; break; case "Processors Type": Category type; Enum.TryParse <Category>(rowValue, out type); cpu.Category = type; break; case "Series": var seriesName = rowValue.Replace("Series", string.Empty).Trim(); var series = this.seriesRepo.All().FirstOrDefault(x => x.Name == seriesName); if (series == null) { series = new Series { Name = seriesName, }; } cpu.Series = series; break; case "CPU Socket Type": var socketName = rowValue.Replace("Socket ", string.Empty); var socket = this.socketRepo.All().FirstOrDefault(x => x.Name == socketName); if (socket == null) { socket = new Socket { Name = socketName, }; } cpu.Socket = socket; break; case "Core Name": var coreName = this.corenameRepo.All().FirstOrDefault(x => x.Name == rowValue); if (coreName == null) { coreName = new CoreName { Name = rowValue, }; } cpu.CoreName = coreName; break; case "# of Cores": var matchResult = this.MatchOneOrMoreDigits.Match(rowValue); byte?numOfCores; if (matchResult.Success) { numOfCores = byte.Parse(matchResult.Value); } else { numOfCores = rowValue.ToLower() switch { "single-core" => 1, "dual-core" => 2, "triple-core" => 3, "quad-core" => 4, "hexa-core" => 6, "octa-core" => 8, "10-core" => 10, "12-core" => 12, "16-core" => 16, _ => null, }; } cpu.Cores = numOfCores; break; case "# of Threads": var numOfTreads = this.MatchOneOrMoreDigits.Match(rowValue).Value; cpu.Threads = short.Parse(numOfTreads); break; case "Operating Frequency": var frequency = this.GetFrecuencyAsShort(rowValue); cpu.Frequency = frequency; break; case "Max Turbo Frequency": var turboFrequency = this.GetFrecuencyAsShort(rowValue); cpu.TurboFrequency = turboFrequency; break; case "L1 Cache": var l1Cache = this.GetCacheAsInt(rowValue); cpu.L1Cache = l1Cache; break; case "L2 Cache": var l2Cache = this.GetCacheAsInt(rowValue); cpu.L2Cache = l2Cache; break; case "L3 Cache": var l3Cache = this.GetCacheAsInt(rowValue); cpu.L3Cache = l3Cache; break; case "Manufacturing Tech": var litography = this.litographyrRepo.All().FirstOrDefault(x => x.Name == rowValue); if (litography == null) { litography = new Lithography { Name = rowValue, }; } cpu.Lithography = litography; break; case "64-Bit Support": var sixtyFourBitSupport = rowValue.Contains("Yes"); cpu.SixtyFourBitSupport = sixtyFourBitSupport; break; case "Memory Types": var typeMatch = new Regex(@"DDR\d\w?").Match(rowValue); if (!typeMatch.Success) { continue; } var memoryTypeAsString = typeMatch.Value; var memoryType = this.memoryTypeRepo.All().FirstOrDefault(x => x.Type == memoryTypeAsString); if (memoryType == null) { memoryType = new MemoryType { Type = memoryTypeAsString, }; } var match = new Regex(@"\d{3,4}").Match(rowValue); if (match.Success) { var memorySpeedAsShort = short.Parse(match.Value); var memorySpeed = this.memorySpeedRepo.All().FirstOrDefault(x => x.Speed == memorySpeedAsShort); if (memorySpeed == null) { memorySpeed = new MemorySpeed { Speed = memorySpeedAsShort, }; } cpu.MemorySpeed = memorySpeed; } cpu.MemoryType = memoryType; break; case "Memory Channel": cpu.MemoryChannel = byte.Parse(rowValue); break; case "Virtualization Technology Support": cpu.VirtualizationSupport = rowValue.Contains("Yes"); break; case "Integrated Graphics": integratedGrapicsData["Name"] = rowValue; break; case "Graphics Base Frequency": integratedGrapicsData["BaseFrequency"] = rowValue; break; case "Graphics Max Dynamic Frequency": integratedGrapicsData["MaxFrequency"] = rowValue; break; case "PCI Express Revision": cpu.PCIERevision = float.Parse(rowValue, CultureInfo.InvariantCulture); break; case "Max Number of PCI Express Lanes": cpu.PCIELanes = byte.Parse(rowValue); break; case "Thermal Design Power": var thermalDesignPower = this.MatchOneOrMoreDigits.Match(rowValue).Value; cpu.ThermalDesignPower = short.Parse(thermalDesignPower); break; case "Cooling Device": cpu.HasCoolingDevice = !rowValue.Contains("not included"); break; case "Date First Available": cpu.FirstAvailable = DateTime.Parse(rowValue); break; default: break; } } if (integratedGrapicsData.ContainsKey("Name")) { var name = integratedGrapicsData["Name"]; var integratedGrapic = this.integratedGraphicRepo.All() .FirstOrDefault(x => x.Name == name); if (integratedGrapic == null) { integratedGrapic = new IntegratedGraphic { Name = name, }; if (integratedGrapicsData.ContainsKey("BaseFrequency")) { integratedGrapic.BaseFrequency = this.GetFrecuencyAsShort(integratedGrapicsData["BaseFrequency"]); } if (integratedGrapicsData.ContainsKey("MaxFrequency")) { integratedGrapic.MaxFrequency = this.GetFrecuencyAsShort(integratedGrapicsData["MaxFrequency"]); } } cpu.IntegratedGraphic = integratedGrapic; } if (string.IsNullOrEmpty(cpu.Name) || string.IsNullOrEmpty(cpu.Model)) { var message = "Invalid Name or Model."; this.logger.LogWarning(message); return(message); } await this.cpuRepo.AddAsync(cpu); await this.cpuRepo.SaveChangesAsync(); var successMessage = $"Successfully added {cpu.Model}."; this.logger.LogInformation(successMessage); return(successMessage); }
public async Task <string> ScrapeFromProductPageAsync(string productUrl) { if (productUrl.Contains("Combo")) { var message = "Invalid Product."; this.logger.LogWarning(message); return(message); } var document = await this.Context.OpenAsync(productUrl); var memoryDataTableRows = this.GetAllTablesRows(document); var memoryDataTables = this.GetAllTables(document); var memory = new Memory { Price = this.GetPrice(document), ImageUrl = this.GetImageUrl(document), Category = this.GetCategoryFromUrl(productUrl), }; this.logger.LogInformation(productUrl); foreach (var tableRow in memoryDataTableRows) { var rowName = tableRow.FirstChild.TextContent.Trim(); var rowValue = tableRow.LastElementChild.InnerHtml.Replace("<br><br>", "{n}").Replace("<br>", "{n}").Trim(); switch (rowName) { case "Model": if (this.memoryRepo.AllAsNoTracking().Any(x => x.Model == rowValue)) { var message = "Already exists."; this.logger.LogWarning(message); return(message); } memory.Model = rowValue; break; case "Brand": var brand = this.brandRepo.All().FirstOrDefault(x => x.Name == rowValue); if (brand == null) { brand = new Brand { Name = rowValue, }; } memory.Brand = brand; break; case "Series": var seriesName = rowValue.Replace("Series", string.Empty).Trim(); var series = this.seriesRepo.All().FirstOrDefault(x => x.Name == seriesName); if (series == null) { series = new Series { Name = seriesName, }; } memory.Series = series; break; case "Capacity": var match = new Regex(@"\((?'quantity'\d)\s?[xX]\s?(?'capacity'\d+)(?'memorySize'[A-Za-z]+)\)").Match(rowValue); if (match.Success) { memory.NumberOfModules = byte.Parse(match.Groups["quantity"].Value); var size = match.Groups["memorySize"].Value; var capacity = int.Parse(match.Groups["capacity"].Value); if (size.ToLower() == "gb") { capacity *= 1024; } memory.CapacityPerModule = capacity; } else { memory.NumberOfModules = 1; var digitMatch = this.MatchOneOrMoreDigits.Match(rowValue); if (!digitMatch.Success) { continue; } var capacity = int.Parse(digitMatch.Value); if (rowValue.ToLower().Contains("gb")) { capacity *= 1024; } memory.CapacityPerModule = capacity; } break; case "Type": var typeMatch = new Regex(@"DDR\d\w?").Match(rowValue); if (!typeMatch.Success) { continue; } var type = this.memoryTypeRepo.All().FirstOrDefault(x => x.Type == typeMatch.Value); if (type == null) { type = new MemoryType { Type = typeMatch.Value, }; } memory.MemoryType = type; break; case "Speed": var speedMatch = new Regex(@"\d{3,4}").Match(rowValue); if (!speedMatch.Success) { continue; } var speed = this.memorySpeedRepo.All().FirstOrDefault(x => x.Speed == short.Parse(speedMatch.Value)); if (speed == null) { speed = new MemorySpeed { Speed = short.Parse(speedMatch.Value), }; } memory.MemorySpeed = speed; break; case "CAS Latency": memory.ColumnAddressStrobeLatency = this.MatchAndParseFloat(rowValue); break; case "Timing": memory.Timings = rowValue; break; case "Voltage": memory.Voltage = this.MatchAndParseFloat(rowValue); break; case "Heat Spreader": if (rowValue.ToLower().Contains("no") || rowValue.ToLower().Contains("none")) { memory.HeatSpreader = false; } else { memory.HeatSpreader = true; } break; case "Features": memory.Features = rowValue; break; case "Date First Available": memory.FirstAvailable = DateTime.Parse(rowValue); break; } } if (memory.Model == null) { var message = "Invalid Model."; this.logger.LogWarning(message); return(message); } await this.memoryRepo.AddAsync(memory); await this.memoryRepo.SaveChangesAsync(); var successMessage = $"Successfully added {memory.Model}."; this.logger.LogInformation(successMessage); return(successMessage); }
public async Task <string> ScrapeFromProductPageAsync(string productUrl) { if (productUrl.Contains("Combo")) { var message = "Invalid Product."; this.logger.LogWarning(message); return(message); } var document = await this.Context.OpenAsync(productUrl); var motherboardDataTableRows = this.GetAllTablesRows(document); var motherboardDataTables = this.GetAllTables(document); var motherboard = new Motherboard { Price = this.GetPrice(document), ImageUrl = this.GetImageUrl(document), Category = this.GetCategoryFromUrl(productUrl), }; foreach (var table in motherboardDataTables) { var caption = table.FirstElementChild.TextContent; var tableRows = table.QuerySelectorAll("tr"); switch (caption) { case "Onboard Audio": var audioChipsetName = tableRows[0].LastChild.TextContent.Trim(); var firstLine = tableRows[1].LastElementChild .InnerHtml.Replace("<br><br>", "{n}").Replace("<br>", "{n}").Trim().Split("{n}"); var matches = this.MatchOneOrMoreDigitsFloat.Matches(firstLine[0]); float channels = 0; foreach (Match match in matches) { channels = Math.Max(channels, float.Parse(match.Value, CultureInfo.InvariantCulture)); } var audioChipset = this.audioChipsetRepo.All().FirstOrDefault(x => x.Name == audioChipsetName); if (audioChipset == null) { audioChipset = new AudioChipset { Name = audioChipsetName, Channels = channels, }; } motherboard.AudioChipset = audioChipset; break; case "Onboard LAN": var lanChipsetName = tableRows[0].LastChild.TextContent.Trim(); var lanChipset = this.lanChipsetRepo.All().FirstOrDefault(x => x.Name == lanChipsetName); if (lanChipset == null) { lanChipset = new LanChipset { Name = lanChipsetName, }; } motherboard.LanChipset = lanChipset; break; case "Rear Panel Ports": var ports = string.Empty; foreach (var tableRow in tableRows) { if (tableRow.FirstChild.TextContent != "Back I/O Ports") { ports += tableRow.LastChild.TextContent.Trim() + Environment.NewLine; } } if (!string.IsNullOrEmpty(ports)) { motherboard.RearPanelPorts = ports; } break; } } this.logger.LogInformation(productUrl); foreach (var tableRow in motherboardDataTableRows) { var rowName = tableRow.FirstChild.TextContent.Trim(); var rowValue = tableRow.LastElementChild.InnerHtml.Replace("<br><br>", "{n}").Replace("<br>", "{n}").Trim(); switch (rowName) { case "Model": if (this.motherboardRepo.AllAsNoTracking().Any(x => x.Model == rowValue)) { var message = "Already exists."; this.logger.LogWarning(message); return(message); } motherboard.Model = rowValue; break; case "Brand": var brand = this.brandRepo.All().FirstOrDefault(x => x.Name == rowValue); if (brand == null) { brand = new Brand { Name = rowValue, }; } motherboard.Brand = brand; break; case "Series": var seriesName = rowValue.Replace("Series", string.Empty).Trim(); var series = this.seriesRepo.All().FirstOrDefault(x => x.Name == seriesName); if (series == null) { series = new Series { Name = seriesName, }; } motherboard.Series = series; break; case "CPU Socket Type": var socket = this.socketRepo.All().FirstOrDefault(x => x.Name == rowValue); if (socket == null) { socket = new Socket { Name = rowValue, }; } motherboard.Socket = socket; break; case "Chipset": var chipset = this.chipsetreopo.All().FirstOrDefault(x => x.Name == rowValue); if (chipset == null) { chipset = new MothrboardChipset { Name = rowValue, }; } motherboard.Chipset = chipset; break; case "Number of Memory Slots": motherboard.MemorySlots = int.Parse(this.MatchOneOrMoreDigits.Match(rowValue).Value); break; case "Memory Standard": var lines = rowValue.Split("{n}"); foreach (var line in lines) { var typeMatch = new Regex(@"DDR\d\w?").Match(line); if (!typeMatch.Success) { continue; } var memoryTypeAsString = typeMatch.Value; var matches = new Regex(@"\d{3,4}").Matches(line); foreach (Match match in matches) { var memorySpeedAsShort = short.Parse(match.Value); var motherboardMemoryType = this.motherboardMemoryTypeRepo.All() .FirstOrDefault(x => x.MotherboardId == motherboard.Id && x.MemoryType.Type == memoryTypeAsString && x.MemorySpeed.Speed == memorySpeedAsShort); if (motherboardMemoryType != null) { continue; } var memoryType = this.memoryTypeRepo.All().FirstOrDefault(x => x.Type == memoryTypeAsString); if (memoryType == null) { memoryType = new MemoryType { Type = memoryTypeAsString, }; await this.memoryTypeRepo.AddAsync(memoryType); await this.memoryTypeRepo.SaveChangesAsync(); } var memorySpeed = this.memorySpeedRepo.All().FirstOrDefault(x => x.Speed == memorySpeedAsShort); if (memorySpeed == null) { memorySpeed = new MemorySpeed { Speed = memorySpeedAsShort, }; await this.memorySpeedRepo.AddAsync(memorySpeed); await this.memorySpeedRepo.SaveChangesAsync(); } motherboardMemoryType = new MotherboardMemoryType { Motherboard = motherboard, MemoryType = memoryType, MemorySpeed = memorySpeed, }; motherboard.MotherboardMemoryType.Add(motherboardMemoryType); } } break; case "Maximum Memory Supported": var maximumMemory = int.Parse(this.MatchOneOrMoreDigits.Match(rowValue).Value); if (rowValue.ToLower().Contains("gb")) { maximumMemory *= 1024; } motherboard.MaxMemorySupport = maximumMemory; break; case "Channel Supported": byte?channel = rowValue.ToLower() switch { "dual channel" => 2, "triple channel" => 3, "quad channel" => 4, _ => null, }; motherboard.MemoryChannel = channel; break; case "PCI Express 3.0 x16": motherboard.PCIe3x16 = this.GetNumberOfSlots(@"(?'slots'\d) x PCI", rowValue); break; case "PCI Express 4.0 x16": motherboard.PCIe4x16 = this.GetNumberOfSlots(@"(?'slots'\d) x PCI", rowValue); break; case "PCI Express x1": motherboard.PCIex1 = this.GetNumberOfSlots(@"(?'slots'\d) x PCI", rowValue); break; case "SATA 6Gb/s": motherboard.Sata6Gbs = this.GetNumberOfSlots(@"(?'slots'\d) x SATA", rowValue); break; case "SATA 3Gb/s": motherboard.Sata3Gbs = this.GetNumberOfSlots(@"(?'slots'\d) x SATA", rowValue); break; case "M.2": motherboard.Mdot2 = this.GetNumberOfSlots(@"(?'slots'\d) x M\.", rowValue); break; case "Back I/O Ports": motherboard.RearPanelPorts = rowValue.Replace("\n", "NewLine"); break; case "Form Factor": var formFactor = this.formFactorRepo.All().FirstOrDefault(x => x.Name == rowValue); if (formFactor == null) { formFactor = new FormFactor { Name = rowValue, }; } motherboard.FormFactor = formFactor; break; case "Dimensions (W x L)": var dimensionsSplit = rowValue.Split('x'); var widthInInch = this.MatchAndParseFloat(dimensionsSplit[0]); var lengthInInch = this.MatchAndParseFloat(dimensionsSplit[1]); motherboard.Width = widthInInch * 2.54F; motherboard.Length = lengthInInch * 2.54F; break; case "Features": motherboard.Features = rowValue; break; case "Date First Available": motherboard.FirstAvailable = DateTime.Parse(rowValue); break; } } if (motherboard.Model == null) { var message = "Invalid Model."; this.logger.LogWarning(message); return(message); } await this.motherboardRepo.AddAsync(motherboard); await this.motherboardRepo.SaveChangesAsync(); var successMessage = $"Successfully added {motherboard.Model}."; this.logger.LogInformation(successMessage); return(successMessage); }