public RedirectToRouteResult CreateProductParser(ProductParserCreator model) { var uri = new Uri(model.ParseProductLink); string host = uri.GetLeftPart(UriPartial.Authority); var parseInfo = context.ParseInfoes.FirstOrDefault(info => info.Parselink.Contains(host) && info.IsActive==true); if (parseInfo == null) { var productInfo = new ParseInfo { Name = model.ProductNameXpath, PriceXPath = model.PriceXpath, ImageXpath = model.ImageXpath, Parselink = model.ParseProductLink, CPUXPath = model.CPUXpath, DisplayXPath = model.DisplayXpath, HDDXPath = model.HDDXpath, RAMXPath = model.RAMXpath, VGAXPath = model.VGAXpath, IsActive = true, }; //Create parser if not exist context.ParseInfoes.Add(productInfo); context.SaveChanges(); } else { //If exist Update Parser parseInfo.Name = model.ProductNameXpath; parseInfo.PriceXPath = model.PriceXpath; parseInfo.ImageXpath = model.ImageXpath; parseInfo.Parselink = model.ParseProductLink; parseInfo.CPUXPath = model.CPUXpath; parseInfo.DisplayXPath = model.DisplayXpath; parseInfo.HDDXPath = model.HDDXpath; parseInfo.RAMXPath = model.RAMXpath; parseInfo.VGAXPath = model.VGAXpath; parseInfo.IsActive = true; context.SaveChanges(); } Task.Factory.StartNew(() => ParserHelper.ParseProductData(model)); int rcmId = Int32.Parse(model.RecommendProductId); var recommendProduct = context.RecommendProducts.Where(x => x.ID == rcmId).FirstOrDefault(); recommendProduct.IsApprove = true; context.SaveChanges(); TempData["createbyRecommendProduct"] = "success"; System.Threading.Thread.Sleep(5000); return RedirectToAction("Index"); }
public RedirectToRouteResult CreateProductParserAdmin(ProductParserCreator model) { var parser = new ParseInfo { Name = model.ProductNameXpath, PriceXPath = model.PriceXpath, ImageXpath = model.ImageXpath, Parselink = model.ParseProductLink, CPUXPath = model.CPUXpath, VGAXPath = model.VGAXpath, HDDXPath = model.HDDXpath, DisplayXPath = model.DisplayXpath, RAMXPath = model.RAMXpath }; context.ParseInfoes.Add(parser); context.SaveChanges(); TempData["create"] = "Success"; return RedirectToAction("Index"); }
public static void ParseProductData(ProductParserCreator model) { ConstantManager.IsRecommendRunning = true; // Create Firefox browser var web = new HtmlWeb { UserAgent = "Mozilla/5.0 (Windows NT 6.1; rv:26.0) Gecko/20100101 Firefox/26.0" }; //do more to get data var uri = new Uri(model.ParseProductLink); string host = uri.GetLeftPart(UriPartial.Authority); using (var context = new CPS_SolutionEntities()) { var parseinfo = context.ParseInfoes.Where(p => p.IsActive && p.Parselink.Contains(host)).FirstOrDefault(); var data = GetProductData(web, parseinfo); InsertProductToDb(data, model); ConstantManager.IsRecommendRunning = false; } }
public static void InsertProductToDb(ProductData data, ProductParserCreator model) { //create list of Att List<Hardware> listAttDic = new List<Hardware>(); if (data != null) { //Convert Price double valPrice = 0; if (!String.IsNullOrEmpty(data.Price)) { valPrice = PriceHelper.ConvertPrice(data.Price); } else { valPrice = 0; } if (data.Image.Contains("default.jpg")) { data.Image = "Images/I/default.jpg"; } if (!String.IsNullOrEmpty(data.Name) && !String.IsNullOrEmpty(data.CPU) && !String.IsNullOrEmpty(data.HDD) && !String.IsNullOrEmpty(data.RAM) && !String.IsNullOrEmpty(data.VGA) && !String.IsNullOrEmpty(data.Display)) { using (var context = new CPS_SolutionEntities()) { //Add table product var prod = new Product { Description = "Chưa cập nhật ", ImageURL = data.Image, TotalWeightPoint = 0, IsActive = null, }; //Check for Store var store = context.Stores.Where(x => model.ParseProductLink.Contains(x.StoreUrl)).FirstOrDefault(); int StoreID = 1; string patter = "://|/"; Regex reg = new Regex(patter); string host = reg.Split(model.ParseProductLink)[1]; if (store != null) { StoreID = store.ID; } else { var newStore = new Store { IsActive = false, LogoImage = "default", StoreUrl = host, StoreName = "Chưa xác định", }; context.Stores.Add(newStore); context.SaveChanges(); StoreID = newStore.ID; } int brand = 13; var allBrands = context.Brands.ToList(); foreach (var item in allBrands) { if (model.ParseProductLink.ToUpper().Contains(item.BrandName.ToUpper())) { brand = item.ID; } else if (model.ParseProductLink.ToUpper().Contains("MACBOOK")) { brand = 3; } else { brand = 12; } } prod.AliasProducts.Add(new AliasProduct() { Name = data.Name, URL = model.ParseProductLink, Price = valPrice, StoreID = StoreID, IsMain = true, IsActive = true, UpdateTime = DateTime.Now, BrandID = brand }); context.Products.Add(prod); context.SaveChanges(); List<KeyValuePair<string, string>> listofAttributes = new List<KeyValuePair<string, string>>(); var cpu = new KeyValuePair<string, string>(data.CPU, "C"); var ram = new KeyValuePair<string, string>(data.RAM, "R"); var vga = new KeyValuePair<string, string>(data.VGA, "V"); var hdd = new KeyValuePair<string, string>(data.HDD, "H"); var display = new KeyValuePair<string, string>(data.Display, "D"); listofAttributes.Add(cpu); listofAttributes.Add(ram); listofAttributes.Add(hdd); listofAttributes.Add(vga); listofAttributes.Add(display); int count = 0; //add 5 attribute for 1 product foreach (var attribute in listofAttributes) { ////Check good match var goodMatch = new List<int>(); var averageMatch = new List<int>(); int pId = -1; bool wholeMatch = false; string codeType = "C"; if (count == 1) { codeType = "R"; } else if (count == 2) { codeType = "H"; } else if (count == 3) { codeType = "V"; } else if (count == 4) { codeType = "D"; } count++; foreach (var alias in context.Dictionaries.Where(x => x.Hardware.IsActive == true && x.Hardware.CodetypeID.Equals(codeType))) { if (attribute.Key == alias.Name) { wholeMatch = true; pId = alias.AttributeDicID; break; } double matchPercent = CompareStringHelper.CompareString(attribute.Key, alias.Name); //good match if (matchPercent > 90) { goodMatch.Add(alias.AttributeDicID); } // normal Match if (matchPercent > 80) { averageMatch.Add(alias.AttributeDicID); } } if (!wholeMatch) { if (goodMatch.Count == 1) { // Match well with only 1 product, take it pId = goodMatch[0]; } else if (goodMatch.Count > 1) { // Match well with more than 1 product, admin decide ExportTrainingFileForProduct(goodMatch, attribute.Key, prod.ID); continue; } else if (averageMatch.Count > 0 && pId == -1) { // Only average match, admin decide ExportTrainingFileForProduct(averageMatch, attribute.Key, prod.ID); continue; } } // If attDic alr Existed? if (pId != -1) { var productAtt = new ProductAttribute { ProductID = prod.ID, AttributeID = pId, IsActive = true }; context.ProductAttributes.Add(productAtt); context.SaveChanges(); } else { //Add a new record var newADitem = new Hardware { Name = attribute.Key, CodetypeID = attribute.Value, WeightCriteraPoint = 0 }; // Add new item for Product Attribute var productAtt = new ProductAttribute { ProductID = prod.ID, AttributeID = newADitem.ID, }; try { // Save change into DB context.Hardwares.Add(newADitem); context.ProductAttributes.Add(productAtt); context.SaveChanges(); var aliasDic = new Dictionary { AttributeDicID = newADitem.ID, Name = newADitem.Name, IsActive = true, }; context.Dictionaries.Add(aliasDic); context.SaveChanges(); } catch (DbUpdateException) { // Do nothing } } } } } } }
public void DoTask(List<RecommendProduct> listOfRecommend, List<ParseInfo> listOfParseInfo) { string patter = "://|/"; Regex reg = new Regex(patter); List<ParseInfo> listInfo = new List<ParseInfo>(); ParseInfo infoItem = null; List<RecommendProduct> list = new List<RecommendProduct>(); if (listOfRecommend != null) { foreach (var info in listOfParseInfo) { infoItem = new ParseInfo { CPUXPath = info.CPUXPath, DisplayXPath = info.DisplayXPath, HDDXPath = info.HDDXPath, ImageXpath = info.ImageXpath, IsActive = true, Name = info.Name, Parselink = info.Parselink, RAMXPath = info.RAMXPath, VGAXPath = info.VGAXPath, }; listInfo.Add(infoItem); } var fitlerList = listInfo.Select(x => new { host = reg.Split(x.Parselink)[1], x.CPUXPath, x.DisplayXPath, x.HDDXPath, x.ImageXpath, x.Name, x.Parselink, x.RAMXPath, x.VGAXPath, x.IsActive }) .Distinct() .GroupBy(y => new { y.host }) .ToArray(); ProductParserCreator model = null; foreach (var item in fitlerList) { foreach (var recommend in listOfRecommend) { if (recommend.Parselink.Contains(item.Key.host)) { model = new ProductParserCreator { CPUXpath = item.First().CPUXPath, DisplayXpath = item.First().DisplayXPath, HDDXpath = item.First().HDDXPath, ImageXpath = item.First().ImageXpath, IsActive = true, ParseProductLink = item.First().Parselink, ProductNameXpath = item.First().Name, RAMXpath = item.First().RAMXPath, VGAXpath = item.First().VGAXPath, RecommendProductId = recommend.ID.ToString(), }; ParserHelper.ParseProductData(model); int rcmId = Int32.Parse(model.RecommendProductId); var recommendProduct = databaseContext.RecommendProducts.Where(x => x.ID == rcmId).FirstOrDefault(); recommendProduct.IsApprove = true; } } } databaseContext.SaveChanges(); } }
public PartialViewResult LoadPreview(string parseLink,string name,string price,string hdd, string vga,string ram,string cpu,string display, string image) { var model = new ProductParserCreator { ParseProductLink = parseLink, ProductNameXpath = name, PriceXpath = price, ImageXpath = image, HDDXpath =hdd, VGAXpath = vga, CPUXpath = cpu, RAMXpath =ram, DisplayXpath = display }; ProductData data = new ProductData(); try { // Create Firefox browser var web = new HtmlWeb { UserAgent = "Mozilla/5.0 (Windows NT 6.1; rv:26.0) Gecko/20100101 Firefox/26.0" }; //do more to get data var uri = new Uri(model.ParseProductLink); string host = uri.GetLeftPart(UriPartial.Authority); //load page System.Net.ServicePointManager.Expect100Continue = false; HtmlNode.ElementsFlags.Remove("form"); var doc = web.Load(model.ParseProductLink); data = ParserHelper.MatchingProductDataPreview(host, doc, model.ProductNameXpath, model.PriceXpath, model.ImageXpath, model.CPUXpath, model.VGAXpath, model.HDDXpath, model.RAMXpath, model.DisplayXpath); return PartialView(data); } catch (System.Net.WebException ex) { LoadPreview(parseLink, name, price, hdd, vga, ram, cpu, display, image); } catch (HtmlWebException ex) { LoadPreview(parseLink, name, price, hdd, vga, ram, cpu, display, image); } return PartialView(); }