public RedirectToRouteResult CreateProductParser(ProductParserCreator model)
{
var uri = new Uri(model.ParseProductLink);
string host = uri.GetLeftPart(UriPartial.Authority);
var parseInfo = context.ParseInfoes.FirstOrDefault(info => info.Parselink.Contains(host) && info.IsActive==true);
if (parseInfo == null)
{
var productInfo = new ParseInfo
{
Name = model.ProductNameXpath,
PriceXPath = model.PriceXpath,
ImageXpath = model.ImageXpath,
Parselink = model.ParseProductLink,
CPUXPath = model.CPUXpath,
DisplayXPath = model.DisplayXpath,
HDDXPath = model.HDDXpath,
RAMXPath = model.RAMXpath,
VGAXPath = model.VGAXpath,
IsActive = true,
};
//Create parser if not exist
context.ParseInfoes.Add(productInfo);
context.SaveChanges();
}
else
{
//If exist Update Parser
parseInfo.Name = model.ProductNameXpath;
parseInfo.PriceXPath = model.PriceXpath;
parseInfo.ImageXpath = model.ImageXpath;
parseInfo.Parselink = model.ParseProductLink;
parseInfo.CPUXPath = model.CPUXpath;
parseInfo.DisplayXPath = model.DisplayXpath;
parseInfo.HDDXPath = model.HDDXpath;
parseInfo.RAMXPath = model.RAMXpath;
parseInfo.VGAXPath = model.VGAXpath;
parseInfo.IsActive = true;
context.SaveChanges();
}
Task.Factory.StartNew(() => ParserHelper.ParseProductData(model));
int rcmId = Int32.Parse(model.RecommendProductId);
var recommendProduct = context.RecommendProducts.Where(x => x.ID == rcmId).FirstOrDefault();
recommendProduct.IsApprove = true;
context.SaveChanges();
TempData["createbyRecommendProduct"] = "success";
System.Threading.Thread.Sleep(5000);
return RedirectToAction("Index");
}
public RedirectToRouteResult CreateProductParserAdmin(ProductParserCreator model)
{
var parser = new ParseInfo
{
Name = model.ProductNameXpath,
PriceXPath = model.PriceXpath,
ImageXpath = model.ImageXpath,
Parselink = model.ParseProductLink,
CPUXPath = model.CPUXpath,
VGAXPath = model.VGAXpath,
HDDXPath = model.HDDXpath,
DisplayXPath = model.DisplayXpath,
RAMXPath = model.RAMXpath
};
context.ParseInfoes.Add(parser);
context.SaveChanges();
TempData["create"] = "Success";
return RedirectToAction("Index");
}
public static void ParseProductData(ProductParserCreator model)
{
ConstantManager.IsRecommendRunning = true;
// Create Firefox browser
var web = new HtmlWeb { UserAgent = "Mozilla/5.0 (Windows NT 6.1; rv:26.0) Gecko/20100101 Firefox/26.0" };
//do more to get data
var uri = new Uri(model.ParseProductLink);
string host = uri.GetLeftPart(UriPartial.Authority);
using (var context = new CPS_SolutionEntities())
{
var parseinfo = context.ParseInfoes.Where(p => p.IsActive && p.Parselink.Contains(host)).FirstOrDefault();
var data = GetProductData(web, parseinfo);
InsertProductToDb(data, model);
ConstantManager.IsRecommendRunning = false;
}
}
public static void InsertProductToDb(ProductData data, ProductParserCreator model)
{
//create list of Att
List<Hardware> listAttDic = new List<Hardware>();
if (data != null)
{
//Convert Price
double valPrice = 0;
if (!String.IsNullOrEmpty(data.Price))
{
valPrice = PriceHelper.ConvertPrice(data.Price);
}
else
{
valPrice = 0;
}
if (data.Image.Contains("default.jpg"))
{
data.Image = "Images/I/default.jpg";
}
if (!String.IsNullOrEmpty(data.Name) && !String.IsNullOrEmpty(data.CPU) &&
!String.IsNullOrEmpty(data.HDD) && !String.IsNullOrEmpty(data.RAM) &&
!String.IsNullOrEmpty(data.VGA) && !String.IsNullOrEmpty(data.Display))
{
using (var context = new CPS_SolutionEntities())
{
//Add table product
var prod = new Product
{
Description = "Chưa cập nhật ",
ImageURL = data.Image,
TotalWeightPoint = 0,
IsActive = null,
};
//Check for Store
var store = context.Stores.Where(x => model.ParseProductLink.Contains(x.StoreUrl)).FirstOrDefault();
int StoreID = 1;
string patter = "://|/";
Regex reg = new Regex(patter);
string host = reg.Split(model.ParseProductLink)[1];
if (store != null)
{
StoreID = store.ID;
}
else
{
var newStore = new Store
{
IsActive = false,
LogoImage = "default",
StoreUrl = host,
StoreName = "Chưa xác định",
};
context.Stores.Add(newStore);
context.SaveChanges();
StoreID = newStore.ID;
}
int brand = 13;
var allBrands = context.Brands.ToList();
foreach (var item in allBrands)
{
if (model.ParseProductLink.ToUpper().Contains(item.BrandName.ToUpper()))
{
brand = item.ID;
}
else if (model.ParseProductLink.ToUpper().Contains("MACBOOK"))
{
brand = 3;
}
else
{
brand = 12;
}
}
prod.AliasProducts.Add(new AliasProduct()
{
Name = data.Name,
URL = model.ParseProductLink,
Price = valPrice,
StoreID = StoreID,
IsMain = true,
IsActive = true,
UpdateTime = DateTime.Now,
BrandID = brand
});
context.Products.Add(prod);
context.SaveChanges();
List<KeyValuePair<string, string>> listofAttributes = new List<KeyValuePair<string, string>>();
var cpu = new KeyValuePair<string, string>(data.CPU, "C");
var ram = new KeyValuePair<string, string>(data.RAM, "R");
var vga = new KeyValuePair<string, string>(data.VGA, "V");
var hdd = new KeyValuePair<string, string>(data.HDD, "H");
var display = new KeyValuePair<string, string>(data.Display, "D");
listofAttributes.Add(cpu);
listofAttributes.Add(ram);
listofAttributes.Add(hdd);
listofAttributes.Add(vga);
listofAttributes.Add(display);
int count = 0;
//add 5 attribute for 1 product
foreach (var attribute in listofAttributes)
{
////Check good match
var goodMatch = new List<int>();
var averageMatch = new List<int>();
int pId = -1;
bool wholeMatch = false;
string codeType = "C";
if (count == 1)
{
codeType = "R";
}
else if (count == 2)
{
codeType = "H";
}
else if (count == 3)
{
codeType = "V";
}
else if (count == 4)
{
codeType = "D";
}
count++;
foreach (var alias in context.Dictionaries.Where(x => x.Hardware.IsActive == true && x.Hardware.CodetypeID.Equals(codeType)))
{
if (attribute.Key == alias.Name)
{
wholeMatch = true;
pId = alias.AttributeDicID;
break;
}
double matchPercent = CompareStringHelper.CompareString(attribute.Key, alias.Name);
//good match
if (matchPercent > 90)
{
goodMatch.Add(alias.AttributeDicID);
}
// normal Match
if (matchPercent > 80)
{
averageMatch.Add(alias.AttributeDicID);
}
}
if (!wholeMatch)
{
if (goodMatch.Count == 1)
{
// Match well with only 1 product, take it
pId = goodMatch[0];
}
else if (goodMatch.Count > 1)
{
// Match well with more than 1 product, admin decide
ExportTrainingFileForProduct(goodMatch, attribute.Key, prod.ID);
continue;
}
else if (averageMatch.Count > 0 && pId == -1)
{
// Only average match, admin decide
ExportTrainingFileForProduct(averageMatch, attribute.Key, prod.ID);
continue;
}
}
// If attDic alr Existed?
if (pId != -1)
{
var productAtt = new ProductAttribute
{
ProductID = prod.ID,
AttributeID = pId,
IsActive = true
};
context.ProductAttributes.Add(productAtt);
context.SaveChanges();
}
else
{
//Add a new record
var newADitem = new Hardware { Name = attribute.Key, CodetypeID = attribute.Value, WeightCriteraPoint = 0 };
// Add new item for Product Attribute
var productAtt = new ProductAttribute
{
ProductID = prod.ID,
AttributeID = newADitem.ID,
};
try
{
// Save change into DB
context.Hardwares.Add(newADitem);
context.ProductAttributes.Add(productAtt);
context.SaveChanges();
var aliasDic = new Dictionary
{
AttributeDicID = newADitem.ID,
Name = newADitem.Name,
IsActive = true,
};
context.Dictionaries.Add(aliasDic);
context.SaveChanges();
}
catch (DbUpdateException)
{
// Do nothing
}
}
}
}
}
}
}
public void DoTask(List<RecommendProduct> listOfRecommend, List<ParseInfo> listOfParseInfo)
{
string patter = "://|/";
Regex reg = new Regex(patter);
List<ParseInfo> listInfo = new List<ParseInfo>();
ParseInfo infoItem = null;
List<RecommendProduct> list = new List<RecommendProduct>();
if (listOfRecommend != null)
{
foreach (var info in listOfParseInfo)
{
infoItem = new ParseInfo
{
CPUXPath = info.CPUXPath,
DisplayXPath = info.DisplayXPath,
HDDXPath = info.HDDXPath,
ImageXpath = info.ImageXpath,
IsActive = true,
Name = info.Name,
Parselink = info.Parselink,
RAMXPath = info.RAMXPath,
VGAXPath = info.VGAXPath,
};
listInfo.Add(infoItem);
}
var fitlerList = listInfo.Select(x => new
{
host = reg.Split(x.Parselink)[1],
x.CPUXPath,
x.DisplayXPath,
x.HDDXPath,
x.ImageXpath,
x.Name,
x.Parselink,
x.RAMXPath,
x.VGAXPath,
x.IsActive
})
.Distinct()
.GroupBy(y => new { y.host })
.ToArray();
ProductParserCreator model = null;
foreach (var item in fitlerList)
{
foreach (var recommend in listOfRecommend)
{
if (recommend.Parselink.Contains(item.Key.host))
{
model = new ProductParserCreator
{
CPUXpath = item.First().CPUXPath,
DisplayXpath = item.First().DisplayXPath,
HDDXpath = item.First().HDDXPath,
ImageXpath = item.First().ImageXpath,
IsActive = true,
ParseProductLink = item.First().Parselink,
ProductNameXpath = item.First().Name,
RAMXpath = item.First().RAMXPath,
VGAXpath = item.First().VGAXPath,
RecommendProductId = recommend.ID.ToString(),
};
ParserHelper.ParseProductData(model);
int rcmId = Int32.Parse(model.RecommendProductId);
var recommendProduct = databaseContext.RecommendProducts.Where(x => x.ID == rcmId).FirstOrDefault();
recommendProduct.IsApprove = true;
}
}
}
databaseContext.SaveChanges();
}
}
public PartialViewResult LoadPreview(string parseLink,string name,string price,string hdd, string vga,string ram,string cpu,string display, string image)
{
var model = new ProductParserCreator
{
ParseProductLink = parseLink,
ProductNameXpath = name,
PriceXpath = price,
ImageXpath = image,
HDDXpath =hdd,
VGAXpath = vga,
CPUXpath = cpu,
RAMXpath =ram,
DisplayXpath = display
};
ProductData data = new ProductData();
try
{ // Create Firefox browser
var web = new HtmlWeb { UserAgent = "Mozilla/5.0 (Windows NT 6.1; rv:26.0) Gecko/20100101 Firefox/26.0" };
//do more to get data
var uri = new Uri(model.ParseProductLink);
string host = uri.GetLeftPart(UriPartial.Authority);
//load page
System.Net.ServicePointManager.Expect100Continue = false;
HtmlNode.ElementsFlags.Remove("form");
var doc = web.Load(model.ParseProductLink);
data = ParserHelper.MatchingProductDataPreview(host, doc, model.ProductNameXpath, model.PriceXpath, model.ImageXpath, model.CPUXpath, model.VGAXpath, model.HDDXpath, model.RAMXpath, model.DisplayXpath);
return PartialView(data);
}
catch (System.Net.WebException ex)
{
LoadPreview(parseLink, name, price, hdd, vga, ram, cpu, display, image);
}
catch (HtmlWebException ex)
{
LoadPreview(parseLink, name, price, hdd, vga, ram, cpu, display, image);
}
return PartialView();
}
