private static async Task SeedAirCoolers(ApplicationDbContext dbContext)
{
using (SqlConnection connection = new SqlConnection(Config.ConnectionString))
{
connection.Open();
var command = new SqlCommand(SqlCommands.SelectAirCoolersSql, connection);
var airCoolers = new List <AirCooler>();
SqlDataReader reader = await command.ExecuteReaderAsync();
try
{
while (reader.Read())
{
var airCooler = new AirCooler
{
Brand = reader["Brand"].ToString(),
CPUSocketCompatibility = reader["CPUSocketCompatibility"].ToString(),
Image = (byte[])reader["Image"],
ImgUrl = reader["ImgUrl"].ToString(),
FanSize = reader["FanSize"].ToString(),
HeatsinkDimensions = reader["HeatsinkDimensions"].ToString(),
Name = reader["Name"].ToString(),
MaxCPUCoolerHeight = reader["MaxCPUCoolerHeight"].ToString(),
PowerConnector = reader["PowerConnector"].ToString(),
RPM = reader["RPM"].ToString(),
Weight = reader["Weight"].ToString(),
CategoryId = 6,
};
airCoolers.Add(airCooler);
}
dbContext.AirCoolers.AddRange(airCoolers);
await dbContext.SaveChangesAsync();
}
catch (Exception)
{
throw;
}
finally
{
reader.Close();
}
}
}
public async Task <IEnumerable <AirCooler> > GatherAirCoolerData()
{
var videoCards = new List <AirCooler>();
var productUrls = new List <string>();
var parser = new HtmlParser();
var client = new HttpClient();
for (int page = 1; page <= 96; page++)
{
Console.Write($"{page} => ");
var url = $"https://www.newegg.com/Desktop-Memory/SubCategory/ID-574/Page-{page}";
string htmlContent = null;
for (var i = 0; i < 10; i++)
{
try
{
var response = await client.GetAsync(url);
htmlContent = await response.Content.ReadAsStringAsync();
break;
}
catch
{
Console.Write('!');
Thread.Sleep(500);
}
}
if (string.IsNullOrWhiteSpace(htmlContent))
{
break;
}
var document = await parser.ParseDocumentAsync(htmlContent);
var elements = document.GetElementsByClassName("item-container ");
if (elements.Length == 0)
{
break;
}
foreach (var element in elements)
{
string pcPartPickerUrl = null;
var options = element.InnerHtml.Split('\n', StringSplitOptions.RemoveEmptyEntries);
foreach (var option in options)
{
if (option.Contains("href=") && option.Contains("item-img"))
{
var productUrlUntrimmed = option.Substring(option.IndexOf('/'));
var productUrl = productUrlUntrimmed.Substring(0, productUrlUntrimmed.Length - 19);
pcPartPickerUrl = "https:" + productUrl;
productUrls.Add(pcPartPickerUrl);
}
}
}
}
int count = 0;
foreach (var url in productUrls)
{
string htmlContent = null;
for (var i = 0; i < 10; i++)
{
try
{
var response = await client.GetAsync(url);
htmlContent = await response.Content.ReadAsStringAsync();
break;
}
catch
{
Console.Write('!');
Thread.Sleep(500);
}
}
Console.WriteLine(count);
count++;
var document = await parser.ParseDocumentAsync(htmlContent);
var manufacturerInfo = document.GetElementById("MfrContact");
string productUrl = string.Empty;
if (manufacturerInfo != null)
{
productUrl = manufacturerInfo.GetElementsByTagName("a")[0].InnerHtml;
}
var productSpecs = document.GetElementById("detailSpecContent");
string productSpecsInnerHtml = string.Empty;
if (productSpecs == null)
{
continue;
}
productSpecsInnerHtml = productSpecs.InnerHtml;
var specs = productSpecsInnerHtml.Split("<dl>", StringSplitOptions.RemoveEmptyEntries);
var productName = string.Empty;
if (specs[0].Contains("<span>"))
{
productName = specs[0].Substring(specs[0].IndexOf("<span>") + 6).Trim();
productName = productName.Substring(0, productName.IndexOf("</span>")).Trim();
}
var videoCard = new AirCooler
{
Name = productName,
ProductUrl = productUrl,
};
var imgHtmlElemnts = document.GetElementsByName("gallery");
string imgHtml = string.Empty;
if (imgHtmlElemnts.Length > 0)
{
imgHtml = imgHtmlElemnts[0].InnerHtml;
imgHtml = imgHtml.Substring(imgHtml.IndexOf("//") + 2);
imgHtml = imgHtml.Substring(0, imgHtml.IndexOf(">") - 1);
}
if (imgHtml.Length > 0)
{
videoCard.ImgUrl = imgHtml;
}
foreach (var spec in specs)
{
if (spec.Contains("<dt>") && spec.Contains("<dd>"))
{
var replaced = spec.Replace("<dt>", "|");
replaced = replaced.Replace("</dt>", "|");
replaced = replaced.Replace("<dd>", "|");
replaced = replaced.Replace("</dd>", "|");
var specsList = replaced.Split('|', StringSplitOptions.RemoveEmptyEntries);
var specName = specsList[0];
var specValue = specsList[1];
if (specName.Contains("a data"))
{
specName = specName.Substring(specName.IndexOf(">") + 1);
specName = specName.Substring(0, specName.IndexOf("<"));
}
switch (specName)
{
case "Brand":
videoCard.Brand = specValue;
break;
case "Fan Size":
videoCard.FanSize = specValue;
break;
case "CPU Socket Compatibility":
videoCard.CPUSocketCompatibility = specValue;
break;
case "RPM":
videoCard.RPM = specValue;
break;
case "Power Connector":
videoCard.PowerConnector = specValue;
break;
case "Max CPU Cooler Height":
videoCard.MaxCPUCoolerHeight = specValue;
break;
case "Heatsink Dimensions":
videoCard.HeatsinkDimensions = specValue;
break;
case "Weight":
videoCard.Weight = specValue;
break;
default:
break;
}
}
}
videoCards.Add(videoCard);
}
return(videoCards);
}
private static void AddImagesToThePartsCpuToHdd()
{
using (var context = new ApplicationDbContext())
{
var cpus = context.Cpus
.Where(x => x.Name != " ").ToList();
context.SaveChanges();
var cpusToAdd = new List <Cpu>();
Console.WriteLine(cpus.Count);
foreach (var cpu in cpus)
{
Console.WriteLine(cpu.Id);
var imgArray = TransformImage(cpu.ImgUrl).GetAwaiter().GetResult();
var cpuToAdd = new Cpu
{
Brand = cpu.Brand,
CPUSocketType = cpu.CPUSocketType,
Image = imgArray,
ImgUrl = cpu.ImgUrl,
ManufacturingTech = cpu.ManufacturingTech,
Model = cpu.Model,
Name = cpu.Name,
NumberOfCores = cpu.NumberOfCores,
NumberOfThreads = cpu.NumberOfThreads,
ProcesorType = cpu.ProcesorType,
TDP = cpu.TDP,
Series = cpu.Series,
ProductUrl = cpu.ProductUrl,
};
cpusToAdd.Add(cpuToAdd);
}
context.Cpus.RemoveRange(context.Cpus);
context.Cpus.AddRange(cpusToAdd);
context.SaveChanges();
}
using (var context = new ApplicationDbContext())
{
var airCoolers = context.AirCoolers.Where(x => x.Name != " ").ToList();
context.SaveChanges();
var airCoolersToAdd = new List <AirCooler>();
Console.WriteLine(airCoolers.Count);
foreach (var airCooler in airCoolers)
{
Console.WriteLine(airCooler.Id);
var imgArray = TransformImage(airCooler.ImgUrl).GetAwaiter().GetResult();
var airCoolerToAdd = new AirCooler
{
Brand = airCooler.Brand,
FanSize = airCooler.FanSize,
CPUSocketCompatibility = airCooler.CPUSocketCompatibility,
HeatsinkDimensions = airCooler.HeatsinkDimensions,
Image = imgArray,
ImgUrl = airCooler.ImgUrl,
MaxCPUCoolerHeight = airCooler.MaxCPUCoolerHeight,
Name = airCooler.Name,
PowerConnector = airCooler.PowerConnector,
RPM = airCooler.RPM,
Weight = airCooler.Weight,
ProductUrl = airCooler.ProductUrl,
};
airCoolersToAdd.Add(airCoolerToAdd);
}
context.AirCoolers.RemoveRange(context.AirCoolers);
context.AirCoolers.AddRange(airCoolersToAdd);
context.SaveChanges();
}
using (var context = new ApplicationDbContext())
{
var cases = context.Cases.Where(x => x.Name != " ").ToList();
context.SaveChanges();
var casesToAdd = new List <Case>();
Console.WriteLine(cases.Count);
foreach (var @case in cases)
{
Console.WriteLine(@case.Id);
var imgArray = TransformImage(@case.ImgUrl).GetAwaiter().GetResult();
var caseToAdd = new Case
{
Brand = @case.Brand,
MaxCPUCoolerHeightAllowance = @case.MaxCPUCoolerHeightAllowance,
MaxGPULengthAllowance = @case.MaxGPULengthAllowance,
CaseMaterial = @case.CaseMaterial,
Color = @case.Color,
ExpansionSlots = @case.ExpansionSlots,
Image = imgArray,
ImgUrl = @case.ImgUrl,
MotherboardCompatibility = @case.MotherboardCompatibility,
Name = @case.Name,
SidePanelWindow = @case.SidePanelWindow,
Type = @case.Type,
ProductUrl = @case.ProductUrl,
};
casesToAdd.Add(caseToAdd);
@case.Image = imgArray;
}
context.Cases.RemoveRange(context.Cases);
context.Cases.AddRange(casesToAdd);
context.SaveChanges();
}
using (var context = new ApplicationDbContext())
{
var hdds = context.HardDiskDrives.Where(x => x.Name != " ").ToList();
context.SaveChanges();
var hardDisksToAdd = new List <HardDiskDrive>();
Console.WriteLine(hdds.Count);
foreach (var hdd in hdds)
{
Console.WriteLine(hdd.Id);
var imgArray = TransformImage(hdd.ImgUrl).GetAwaiter().GetResult();
var hardToAdd = new HardDiskDrive
{
Brand = hdd.Brand,
Cache = hdd.Cache,
Capacity = hdd.Capacity,
FormFactor = hdd.FormFactor,
Image = imgArray,
ImgUrl = hdd.ImgUrl,
Interface = hdd.Interface,
Name = hdd.Name,
RPM = hdd.RPM,
Usage = hdd.Usage,
ProductUrl = hdd.ProductUrl,
};
hardDisksToAdd.Add(hardToAdd);
hdd.Image = imgArray;
}
context.HardDiskDrives.RemoveRange(context.HardDiskDrives);
context.HardDiskDrives.AddRange(hardDisksToAdd);
context.SaveChanges();
}
}
