public static string GetMemorySpeed(this MemorySpeed me)
{
switch (me)
{
case MemorySpeed.Fast:
return("1400MHz");
case MemorySpeed.Turbo:
return("1600MHz");
default:
return("1400MHz");
}
}
public async Task <string> ScrapeFromProductPageAsync(string productUrl)
{
if (productUrl.Contains("Combo"))
{
var message = "Invalid Product.";
this.logger.LogWarning(message);
return(message);
}
var document = await this.Context.OpenAsync(productUrl);
var cpuData = this.GetAllTablesRows(document);
var integratedGrapicsData = new Dictionary <string, string>();
var cpu = new CPU
{
Price = this.GetPrice(document),
ImageUrl = this.GetImageUrl(document),
};
this.logger.LogInformation(productUrl);
foreach (var tableRow in cpuData)
{
// this.logger.LogInformation(tr.InnerHtml);
var rowName = tableRow.FirstChild.TextContent.Trim().Replace("<!-- --> ", string.Empty);
var rowValue = tableRow.LastElementChild.InnerHtml.Replace("<br><br>", "{n}").Replace("<br>", "{n}").Trim();
switch (rowName)
{
case "Name":
if (this.cpuRepo.AllAsNoTracking().Any(x => x.Name == rowValue))
{
var message = "Already exists.";
this.logger.LogWarning(message);
return(message);
}
cpu.Name = rowValue;
break;
case "Brand":
var brand = this.brandRepo.All().FirstOrDefault(x => x.Name == rowValue);
if (brand == null)
{
brand = new Brand
{
Name = rowValue,
};
}
cpu.Brand = brand;
break;
case "Model":
if (this.cpuRepo.AllAsNoTracking().Any(x => x.Model == rowValue))
{
var message = "Already exists.";
this.logger.LogWarning(message);
return(message);
}
cpu.Model = rowValue;
break;
case "Processors Type":
Category type;
Enum.TryParse <Category>(rowValue, out type);
cpu.Category = type;
break;
case "Series":
var seriesName = rowValue.Replace("Series", string.Empty).Trim();
var series = this.seriesRepo.All().FirstOrDefault(x => x.Name == seriesName);
if (series == null)
{
series = new Series
{
Name = seriesName,
};
}
cpu.Series = series;
break;
case "CPU Socket Type":
var socketName = rowValue.Replace("Socket ", string.Empty);
var socket = this.socketRepo.All().FirstOrDefault(x => x.Name == socketName);
if (socket == null)
{
socket = new Socket
{
Name = socketName,
};
}
cpu.Socket = socket;
break;
case "Core Name":
var coreName = this.corenameRepo.All().FirstOrDefault(x => x.Name == rowValue);
if (coreName == null)
{
coreName = new CoreName
{
Name = rowValue,
};
}
cpu.CoreName = coreName;
break;
case "# of Cores":
var matchResult = this.MatchOneOrMoreDigits.Match(rowValue);
byte?numOfCores;
if (matchResult.Success)
{
numOfCores = byte.Parse(matchResult.Value);
}
else
{
numOfCores = rowValue.ToLower() switch
{
"single-core" => 1,
"dual-core" => 2,
"triple-core" => 3,
"quad-core" => 4,
"hexa-core" => 6,
"octa-core" => 8,
"10-core" => 10,
"12-core" => 12,
"16-core" => 16,
_ => null,
};
}
cpu.Cores = numOfCores;
break;
case "# of Threads":
var numOfTreads = this.MatchOneOrMoreDigits.Match(rowValue).Value;
cpu.Threads = short.Parse(numOfTreads);
break;
case "Operating Frequency":
var frequency = this.GetFrecuencyAsShort(rowValue);
cpu.Frequency = frequency;
break;
case "Max Turbo Frequency":
var turboFrequency = this.GetFrecuencyAsShort(rowValue);
cpu.TurboFrequency = turboFrequency;
break;
case "L1 Cache":
var l1Cache = this.GetCacheAsInt(rowValue);
cpu.L1Cache = l1Cache;
break;
case "L2 Cache":
var l2Cache = this.GetCacheAsInt(rowValue);
cpu.L2Cache = l2Cache;
break;
case "L3 Cache":
var l3Cache = this.GetCacheAsInt(rowValue);
cpu.L3Cache = l3Cache;
break;
case "Manufacturing Tech":
var litography = this.litographyrRepo.All().FirstOrDefault(x => x.Name == rowValue);
if (litography == null)
{
litography = new Lithography
{
Name = rowValue,
};
}
cpu.Lithography = litography;
break;
case "64-Bit Support":
var sixtyFourBitSupport = rowValue.Contains("Yes");
cpu.SixtyFourBitSupport = sixtyFourBitSupport;
break;
case "Memory Types":
var typeMatch = new Regex(@"DDR\d\w?").Match(rowValue);
if (!typeMatch.Success)
{
continue;
}
var memoryTypeAsString = typeMatch.Value;
var memoryType = this.memoryTypeRepo.All().FirstOrDefault(x => x.Type == memoryTypeAsString);
if (memoryType == null)
{
memoryType = new MemoryType
{
Type = memoryTypeAsString,
};
}
var match = new Regex(@"\d{3,4}").Match(rowValue);
if (match.Success)
{
var memorySpeedAsShort = short.Parse(match.Value);
var memorySpeed = this.memorySpeedRepo.All().FirstOrDefault(x => x.Speed == memorySpeedAsShort);
if (memorySpeed == null)
{
memorySpeed = new MemorySpeed
{
Speed = memorySpeedAsShort,
};
}
cpu.MemorySpeed = memorySpeed;
}
cpu.MemoryType = memoryType;
break;
case "Memory Channel":
cpu.MemoryChannel = byte.Parse(rowValue);
break;
case "Virtualization Technology Support":
cpu.VirtualizationSupport = rowValue.Contains("Yes");
break;
case "Integrated Graphics":
integratedGrapicsData["Name"] = rowValue;
break;
case "Graphics Base Frequency":
integratedGrapicsData["BaseFrequency"] = rowValue;
break;
case "Graphics Max Dynamic Frequency":
integratedGrapicsData["MaxFrequency"] = rowValue;
break;
case "PCI Express Revision":
cpu.PCIERevision = float.Parse(rowValue, CultureInfo.InvariantCulture);
break;
case "Max Number of PCI Express Lanes":
cpu.PCIELanes = byte.Parse(rowValue);
break;
case "Thermal Design Power":
var thermalDesignPower = this.MatchOneOrMoreDigits.Match(rowValue).Value;
cpu.ThermalDesignPower = short.Parse(thermalDesignPower);
break;
case "Cooling Device":
cpu.HasCoolingDevice = !rowValue.Contains("not included");
break;
case "Date First Available":
cpu.FirstAvailable = DateTime.Parse(rowValue);
break;
default:
break;
}
}
if (integratedGrapicsData.ContainsKey("Name"))
{
var name = integratedGrapicsData["Name"];
var integratedGrapic = this.integratedGraphicRepo.All()
.FirstOrDefault(x => x.Name == name);
if (integratedGrapic == null)
{
integratedGrapic = new IntegratedGraphic
{
Name = name,
};
if (integratedGrapicsData.ContainsKey("BaseFrequency"))
{
integratedGrapic.BaseFrequency = this.GetFrecuencyAsShort(integratedGrapicsData["BaseFrequency"]);
}
if (integratedGrapicsData.ContainsKey("MaxFrequency"))
{
integratedGrapic.MaxFrequency = this.GetFrecuencyAsShort(integratedGrapicsData["MaxFrequency"]);
}
}
cpu.IntegratedGraphic = integratedGrapic;
}
if (string.IsNullOrEmpty(cpu.Name) || string.IsNullOrEmpty(cpu.Model))
{
var message = "Invalid Name or Model.";
this.logger.LogWarning(message);
return(message);
}
await this.cpuRepo.AddAsync(cpu);
await this.cpuRepo.SaveChangesAsync();
var successMessage = $"Successfully added {cpu.Model}.";
this.logger.LogInformation(successMessage);
return(successMessage);
}
public async Task <string> ScrapeFromProductPageAsync(string productUrl)
{
if (productUrl.Contains("Combo"))
{
var message = "Invalid Product.";
this.logger.LogWarning(message);
return(message);
}
var document = await this.Context.OpenAsync(productUrl);
var memoryDataTableRows = this.GetAllTablesRows(document);
var memoryDataTables = this.GetAllTables(document);
var memory = new Memory
{
Price = this.GetPrice(document),
ImageUrl = this.GetImageUrl(document),
Category = this.GetCategoryFromUrl(productUrl),
};
this.logger.LogInformation(productUrl);
foreach (var tableRow in memoryDataTableRows)
{
var rowName = tableRow.FirstChild.TextContent.Trim();
var rowValue = tableRow.LastElementChild.InnerHtml.Replace("<br><br>", "{n}").Replace("<br>", "{n}").Trim();
switch (rowName)
{
case "Model":
if (this.memoryRepo.AllAsNoTracking().Any(x => x.Model == rowValue))
{
var message = "Already exists.";
this.logger.LogWarning(message);
return(message);
}
memory.Model = rowValue;
break;
case "Brand":
var brand = this.brandRepo.All().FirstOrDefault(x => x.Name == rowValue);
if (brand == null)
{
brand = new Brand
{
Name = rowValue,
};
}
memory.Brand = brand;
break;
case "Series":
var seriesName = rowValue.Replace("Series", string.Empty).Trim();
var series = this.seriesRepo.All().FirstOrDefault(x => x.Name == seriesName);
if (series == null)
{
series = new Series
{
Name = seriesName,
};
}
memory.Series = series;
break;
case "Capacity":
var match = new Regex(@"\((?'quantity'\d)\s?[xX]\s?(?'capacity'\d+)(?'memorySize'[A-Za-z]+)\)").Match(rowValue);
if (match.Success)
{
memory.NumberOfModules = byte.Parse(match.Groups["quantity"].Value);
var size = match.Groups["memorySize"].Value;
var capacity = int.Parse(match.Groups["capacity"].Value);
if (size.ToLower() == "gb")
{
capacity *= 1024;
}
memory.CapacityPerModule = capacity;
}
else
{
memory.NumberOfModules = 1;
var digitMatch = this.MatchOneOrMoreDigits.Match(rowValue);
if (!digitMatch.Success)
{
continue;
}
var capacity = int.Parse(digitMatch.Value);
if (rowValue.ToLower().Contains("gb"))
{
capacity *= 1024;
}
memory.CapacityPerModule = capacity;
}
break;
case "Type":
var typeMatch = new Regex(@"DDR\d\w?").Match(rowValue);
if (!typeMatch.Success)
{
continue;
}
var type = this.memoryTypeRepo.All().FirstOrDefault(x => x.Type == typeMatch.Value);
if (type == null)
{
type = new MemoryType
{
Type = typeMatch.Value,
};
}
memory.MemoryType = type;
break;
case "Speed":
var speedMatch = new Regex(@"\d{3,4}").Match(rowValue);
if (!speedMatch.Success)
{
continue;
}
var speed = this.memorySpeedRepo.All().FirstOrDefault(x => x.Speed == short.Parse(speedMatch.Value));
if (speed == null)
{
speed = new MemorySpeed
{
Speed = short.Parse(speedMatch.Value),
};
}
memory.MemorySpeed = speed;
break;
case "CAS Latency":
memory.ColumnAddressStrobeLatency = this.MatchAndParseFloat(rowValue);
break;
case "Timing":
memory.Timings = rowValue;
break;
case "Voltage":
memory.Voltage = this.MatchAndParseFloat(rowValue);
break;
case "Heat Spreader":
if (rowValue.ToLower().Contains("no") || rowValue.ToLower().Contains("none"))
{
memory.HeatSpreader = false;
}
else
{
memory.HeatSpreader = true;
}
break;
case "Features":
memory.Features = rowValue;
break;
case "Date First Available":
memory.FirstAvailable = DateTime.Parse(rowValue);
break;
}
}
if (memory.Model == null)
{
var message = "Invalid Model.";
this.logger.LogWarning(message);
return(message);
}
await this.memoryRepo.AddAsync(memory);
await this.memoryRepo.SaveChangesAsync();
var successMessage = $"Successfully added {memory.Model}.";
this.logger.LogInformation(successMessage);
return(successMessage);
}
public async Task <string> ScrapeFromProductPageAsync(string productUrl)
{
if (productUrl.Contains("Combo"))
{
var message = "Invalid Product.";
this.logger.LogWarning(message);
return(message);
}
var document = await this.Context.OpenAsync(productUrl);
var motherboardDataTableRows = this.GetAllTablesRows(document);
var motherboardDataTables = this.GetAllTables(document);
var motherboard = new Motherboard
{
Price = this.GetPrice(document),
ImageUrl = this.GetImageUrl(document),
Category = this.GetCategoryFromUrl(productUrl),
};
foreach (var table in motherboardDataTables)
{
var caption = table.FirstElementChild.TextContent;
var tableRows = table.QuerySelectorAll("tr");
switch (caption)
{
case "Onboard Audio":
var audioChipsetName = tableRows[0].LastChild.TextContent.Trim();
var firstLine = tableRows[1].LastElementChild
.InnerHtml.Replace("<br><br>", "{n}").Replace("<br>", "{n}").Trim().Split("{n}");
var matches = this.MatchOneOrMoreDigitsFloat.Matches(firstLine[0]);
float channels = 0;
foreach (Match match in matches)
{
channels = Math.Max(channels, float.Parse(match.Value, CultureInfo.InvariantCulture));
}
var audioChipset = this.audioChipsetRepo.All().FirstOrDefault(x => x.Name == audioChipsetName);
if (audioChipset == null)
{
audioChipset = new AudioChipset
{
Name = audioChipsetName,
Channels = channels,
};
}
motherboard.AudioChipset = audioChipset;
break;
case "Onboard LAN":
var lanChipsetName = tableRows[0].LastChild.TextContent.Trim();
var lanChipset = this.lanChipsetRepo.All().FirstOrDefault(x => x.Name == lanChipsetName);
if (lanChipset == null)
{
lanChipset = new LanChipset
{
Name = lanChipsetName,
};
}
motherboard.LanChipset = lanChipset;
break;
case "Rear Panel Ports":
var ports = string.Empty;
foreach (var tableRow in tableRows)
{
if (tableRow.FirstChild.TextContent != "Back I/O Ports")
{
ports += tableRow.LastChild.TextContent.Trim() + Environment.NewLine;
}
}
if (!string.IsNullOrEmpty(ports))
{
motherboard.RearPanelPorts = ports;
}
break;
}
}
this.logger.LogInformation(productUrl);
foreach (var tableRow in motherboardDataTableRows)
{
var rowName = tableRow.FirstChild.TextContent.Trim();
var rowValue = tableRow.LastElementChild.InnerHtml.Replace("<br><br>", "{n}").Replace("<br>", "{n}").Trim();
switch (rowName)
{
case "Model":
if (this.motherboardRepo.AllAsNoTracking().Any(x => x.Model == rowValue))
{
var message = "Already exists.";
this.logger.LogWarning(message);
return(message);
}
motherboard.Model = rowValue;
break;
case "Brand":
var brand = this.brandRepo.All().FirstOrDefault(x => x.Name == rowValue);
if (brand == null)
{
brand = new Brand
{
Name = rowValue,
};
}
motherboard.Brand = brand;
break;
case "Series":
var seriesName = rowValue.Replace("Series", string.Empty).Trim();
var series = this.seriesRepo.All().FirstOrDefault(x => x.Name == seriesName);
if (series == null)
{
series = new Series
{
Name = seriesName,
};
}
motherboard.Series = series;
break;
case "CPU Socket Type":
var socket = this.socketRepo.All().FirstOrDefault(x => x.Name == rowValue);
if (socket == null)
{
socket = new Socket
{
Name = rowValue,
};
}
motherboard.Socket = socket;
break;
case "Chipset":
var chipset = this.chipsetreopo.All().FirstOrDefault(x => x.Name == rowValue);
if (chipset == null)
{
chipset = new MothrboardChipset
{
Name = rowValue,
};
}
motherboard.Chipset = chipset;
break;
case "Number of Memory Slots":
motherboard.MemorySlots = int.Parse(this.MatchOneOrMoreDigits.Match(rowValue).Value);
break;
case "Memory Standard":
var lines = rowValue.Split("{n}");
foreach (var line in lines)
{
var typeMatch = new Regex(@"DDR\d\w?").Match(line);
if (!typeMatch.Success)
{
continue;
}
var memoryTypeAsString = typeMatch.Value;
var matches = new Regex(@"\d{3,4}").Matches(line);
foreach (Match match in matches)
{
var memorySpeedAsShort = short.Parse(match.Value);
var motherboardMemoryType = this.motherboardMemoryTypeRepo.All()
.FirstOrDefault(x => x.MotherboardId == motherboard.Id && x.MemoryType.Type == memoryTypeAsString &&
x.MemorySpeed.Speed == memorySpeedAsShort);
if (motherboardMemoryType != null)
{
continue;
}
var memoryType = this.memoryTypeRepo.All().FirstOrDefault(x => x.Type == memoryTypeAsString);
if (memoryType == null)
{
memoryType = new MemoryType
{
Type = memoryTypeAsString,
};
await this.memoryTypeRepo.AddAsync(memoryType);
await this.memoryTypeRepo.SaveChangesAsync();
}
var memorySpeed = this.memorySpeedRepo.All().FirstOrDefault(x => x.Speed == memorySpeedAsShort);
if (memorySpeed == null)
{
memorySpeed = new MemorySpeed
{
Speed = memorySpeedAsShort,
};
await this.memorySpeedRepo.AddAsync(memorySpeed);
await this.memorySpeedRepo.SaveChangesAsync();
}
motherboardMemoryType = new MotherboardMemoryType
{
Motherboard = motherboard,
MemoryType = memoryType,
MemorySpeed = memorySpeed,
};
motherboard.MotherboardMemoryType.Add(motherboardMemoryType);
}
}
break;
case "Maximum Memory Supported":
var maximumMemory = int.Parse(this.MatchOneOrMoreDigits.Match(rowValue).Value);
if (rowValue.ToLower().Contains("gb"))
{
maximumMemory *= 1024;
}
motherboard.MaxMemorySupport = maximumMemory;
break;
case "Channel Supported":
byte?channel = rowValue.ToLower() switch
{
"dual channel" => 2,
"triple channel" => 3,
"quad channel" => 4,
_ => null,
};
motherboard.MemoryChannel = channel;
break;
case "PCI Express 3.0 x16":
motherboard.PCIe3x16 = this.GetNumberOfSlots(@"(?'slots'\d) x PCI", rowValue);
break;
case "PCI Express 4.0 x16":
motherboard.PCIe4x16 = this.GetNumberOfSlots(@"(?'slots'\d) x PCI", rowValue);
break;
case "PCI Express x1":
motherboard.PCIex1 = this.GetNumberOfSlots(@"(?'slots'\d) x PCI", rowValue);
break;
case "SATA 6Gb/s":
motherboard.Sata6Gbs = this.GetNumberOfSlots(@"(?'slots'\d) x SATA", rowValue);
break;
case "SATA 3Gb/s":
motherboard.Sata3Gbs = this.GetNumberOfSlots(@"(?'slots'\d) x SATA", rowValue);
break;
case "M.2":
motherboard.Mdot2 = this.GetNumberOfSlots(@"(?'slots'\d) x M\.", rowValue);
break;
case "Back I/O Ports":
motherboard.RearPanelPorts = rowValue.Replace("\n", "NewLine");
break;
case "Form Factor":
var formFactor = this.formFactorRepo.All().FirstOrDefault(x => x.Name == rowValue);
if (formFactor == null)
{
formFactor = new FormFactor
{
Name = rowValue,
};
}
motherboard.FormFactor = formFactor;
break;
case "Dimensions (W x L)":
var dimensionsSplit = rowValue.Split('x');
var widthInInch = this.MatchAndParseFloat(dimensionsSplit[0]);
var lengthInInch = this.MatchAndParseFloat(dimensionsSplit[1]);
motherboard.Width = widthInInch * 2.54F;
motherboard.Length = lengthInInch * 2.54F;
break;
case "Features":
motherboard.Features = rowValue;
break;
case "Date First Available":
motherboard.FirstAvailable = DateTime.Parse(rowValue);
break;
}
}
if (motherboard.Model == null)
{
var message = "Invalid Model.";
this.logger.LogWarning(message);
return(message);
}
await this.motherboardRepo.AddAsync(motherboard);
await this.motherboardRepo.SaveChangesAsync();
var successMessage = $"Successfully added {motherboard.Model}.";
this.logger.LogInformation(successMessage);
return(successMessage);
}