public void SaveLink(int[] product_ids) { lock (this) { product_ids = product_ids.Distinct().ToArray(); update_category_mapping(product_ids); int link_id = get_minimal_free_link_id(Dbc); //Fhr.ProductOffice.ProductOfficeDataContext db = DbApi.RenewContext(); Dictionary <int, int> company_ids2product_id = new Dictionary <int, int>(); foreach (int product_id in product_ids) { Fhr.ProductOffice.Models.Product p = Db.Products.Where(r => r.Id == product_id).FirstOrDefault(); if (p == null) { continue; } if (company_ids2product_id.ContainsKey(p.CompanyId)) { throw new Exception("Products with Id: " + p.Id + " and " + company_ids2product_id[p.CompanyId] + " belong to the same company: " + p.Company.Name + " and so cannot be linked"); } company_ids2product_id[p.CompanyId] = p.Id; p.LinkId = link_id; } Db.Configuration.ValidateOnSaveEnabled = false; Db.SaveChanges(); } }
internal Product(Engine engine, Fhr.ProductOffice.Models.Product product) { this.engine = engine; if (product == null) throw new Exception("product is null"); DbProduct = product; }
internal Product(Engine engine, Fhr.ProductOffice.Models.Product product) { this.engine = engine; if (product == null) { throw new Exception("product is null"); } DbProduct = product; }
internal Product Get(int product_id) { Product p = null; if (!product_ids2Product.TryGetValue(product_id, out p)) { Fhr.ProductOffice.Models.Product product = engine.Db.Products.Where(x => x.Id == product_id).FirstOrDefault(); if (product == null) { throw new Exception("No Product with Id=" + product_id); } p = new Product(engine, product); product_ids2Product[product.Id] = p; } return(p); }
//public void RenewConfiguration(bool auto_data_analysing) //{ // Configuration_ = new Configuration(this, auto_data_analysing); //} public List <ProductLink> CreateProductLinkList(int[] product1_ids, int company2_id /*, string[] keyword2s = null*/) { lock (this) { if (product1_ids.Length < 1) { throw new Exception("product1_ids is empty"); } Fhr.ProductOffice.Models.Product p1 = Db.Products.Where(p => product1_ids.Contains(p.Id) && p.CompanyId == company2_id).FirstOrDefault(); if (p1 != null) { throw new Exception("Product Id:" + p1.Id + " already belongs to company Id:" + p1.CompanyId + " " + p1.Company.Name + " so no more link can be found."); } //List<int> cis = (from x in Db.Products.Where(p => product1_ids.Contains(p.Id)) join y in Db.Products on x.LinkId equals y.LinkId select y.CompanyId).ToList(); //cis.Add(company2_id); //HashSet<int> cis_ = new HashSet<int>(cis); //foreach (int company_id in cis_) //{ // Configuration.Company c = Configuration.Get(company_id); // if (c.IsDataAnalysisRequired()) // { // PerformDataAnalysis(company_id); // c = new Company(this, company_id); // company_ids2Company[company_id] = c; // } //} Product[] product1s = (from x in product1_ids select Products.Get(x)).ToArray(); List <ProductLink> pls; //if (keyword2s != null && keyword2s.Length > 0) //{ // keyword2s = keyword2s.Where(x => !string.IsNullOrWhiteSpace(x)).Select(x => x.Trim().ToLower()).ToArray(); // HashSet<int> product2_ids = null; // foreach (string keyword2 in keyword2s) // { // HashSet<int> p2_ids = Company.Get(company2_id).Words2ProductIds(Field.Name)[keyword2]; // if (product2_ids == null) // product2_ids = p2_ids; // else // product2_ids = (HashSet<int>)product2_ids.Intersect(p2_ids); // } // List<int> link_ids = (from p2_id in product2_ids let link_id = Product.Get(p2_id).DbProduct.LinkId where link_id > 0 group link_id by link_id into g select (int)g.Key).ToList(); // pls = (from x in link_ids select new ProductLink(product1s, Product.GetLinked(x))).ToList(); // List<int> free_product_ids = (from p2_id in product2_ids let link_id = Product.Get(p2_id).DbProduct.LinkId where link_id == null || link_id <= 0 select p2_id).ToList(); // List<ProductLink> pls2 = (from x in free_product_ids select new ProductLink(product1s, new Product[] { Product.Get(x) })).ToList(); // pls.AddRange(pls2); //} //else //{ //sw1.Start(); List <int> company2_link_ids = new List <int>(); IDataReader dr = Dbc["SELECT LinkId FROM Products WHERE CompanyId=@CompanyId AND LinkId>0 GROUP BY LinkId"].GetReader("@CompanyId", company2_id); //it's faster than LINQ while (dr.Read()) { company2_link_ids.Add((int)dr[0]); } dr.Close(); pls = (from x in company2_link_ids select new ProductLink(this, product1s, Products.GetLinked(x))).ToList(); List <int> free_product_ids = (from p in Companies.Get(company2_id).DbCompany.Products where p.LinkId == null || p.LinkId <= 0 select p.Id).ToList(); List <ProductLink> pls2 = (from x in free_product_ids select new ProductLink(this, product1s, new Product[] { Products.Get(x) })).ToList(); pls.AddRange(pls2); //} //pls = pls.OrderByDescending(x => x.Score).OrderByDescending(x => x.SecondaryScore).ToList(); pls = pls.OrderByDescending(x => x.Score).ToList(); //sw1.Stop(); //string s = "1: " + sw1.ElapsedMilliseconds + ", 2: " + sw2.ElapsedMilliseconds + ", 3: " + sw3.ElapsedMilliseconds + ", 4: " + sw4.ElapsedMilliseconds // + ", 5: " + sw5.ElapsedMilliseconds + ", 6: " + sw6.ElapsedMilliseconds + ", 7: " + sw7.ElapsedMilliseconds // + ", 8: " + sw8.ElapsedMilliseconds + ", 9: " + sw9.ElapsedMilliseconds + ", 10: " + sw10.ElapsedMilliseconds; return(pls); } }
bool process_file(string file, Import import, Action<int> progress=null) { DateTime start_time = DateTime.Now; Excel.IExcelDataReader edr; FileStream stream = System.IO.File.Open(file, FileMode.Open, FileAccess.Read); if (file.EndsWith(".xlsx", StringComparison.InvariantCultureIgnoreCase)) edr = Excel.ExcelReaderFactory.CreateOpenXmlReader(stream); else edr = Excel.ExcelReaderFactory.CreateBinaryReader(stream); //edr = Excel.ExcelReaderFactory.CreateOpenXmlReader(stream); System.Data.DataSet ds = edr.AsDataSet(); edr.Close(); Fhr.ProductOffice.Models.ImportMap import_map = db.ImportMaps.Where(r => r.Id == import.MapId).First(); //if (import_map.C_CompanyProductIdI == null) // throw new Exception("C_CompanyProductIdI in map #" + import_map.Id + " is not specified."); //if (import_map.C_NameI == null) // throw new Exception("C_NameI in map #" + import_map.Id + " is not specified."); db.Configuration.AutoDetectChangesEnabled = false; db.Configuration.ValidateOnSaveEnabled = false; int sheet_count = ds.Tables.Count; int invalid_product_count = 0; int new_product_count = 0; int updated_product_count = 0; int processed_product_count = 0; int invalid_price_count = 0; int new_price_count = 0; int updated_price_count = 0; int processed_price_count = 0; int row_number = -1; List<string> error_messages = new List<string>(); foreach (System.Data.DataTable dt in ds.Tables) { if (import_map.SkipFirstRow) dt.Rows.RemoveAt(0); foreach (System.Data.DataRow row in dt.Rows) { row_number++; Fhr.ProductOffice.Models.Product product = new Fhr.ProductOffice.Models.Product(); product.CompanyId = import_map.CompanyId; product.ExternalId = row[import_map.C_CompanyProductIdI].ToString().Trim(); if (string.IsNullOrWhiteSpace(product.ExternalId)) { bool line_is_empty = true; for (int i = 0; i < 5; i++) if (!string.IsNullOrWhiteSpace(row[i].ToString())) { line_is_empty = false; break; } if (!line_is_empty) { string error = "Cell (" + import_map.C_PriceI + ", " + row_number + ") ExternalId is empty."; if (import.CheckNotImport) { invalid_product_count++; error_messages.Add(error); } else throw new Exception(error); } continue; } product.ModifyTime = null; product.UpdateTime = import.UpdateTime; product.Name = row[import_map.C_NameI].ToString().Trim(); if (product.Name == null) { string error = "Cell (" + import_map.C_PriceI + ", " + row_number + ") Name is empty."; if (import.CheckNotImport) { invalid_product_count++; error_messages.Add(error); } else throw new Exception(error); } product.Source = "file:" + import.File.FileName; if (import_map.C_SkuI != null) product.Sku = row[(int)import_map.C_SkuI].ToString().Trim(); if (import_map.C_CategoryI != null) product.Category = row[(int)import_map.C_CategoryI].ToString().Trim(); if (import_map.C_DescriptionI != null) product.Description = row[(int)import_map.C_DescriptionI].ToString().Trim(); if (!import.CheckNotImport) { Fhr.ProductOffice.Models.Product p = db.Products.Where(r => r.CompanyId == product.CompanyId && r.ExternalId == product.ExternalId).FirstOrDefault(); if (p == null) { product.CreateTime = import.UpdateTime; product.LinkId = null; db.Products.Add(product); new_product_count += db.SaveChanges(); } else { product.Id = p.Id; if (product.Description != null) p.Description = product.Description; if (product.ImageUrls != null) p.ImageUrls = product.ImageUrls; if (product.ModifyTime != null) p.ModifyTime = product.ModifyTime; p.UpdateTime = product.UpdateTime; if (product.Name != null) p.Name = product.Name; if (product.Sku != null) p.Sku = product.Sku; if (product.Source != null) p.Source = product.Source; if (product.Url != null) p.Url = product.Url; //db.Entry(product).State = EntityState.Modified; updated_product_count += db.SaveChanges(); } } processed_product_count++; if (progress != null) progress(processed_product_count); if (import_map.C_PriceI != null) { Price price = new Price(); price.CurrencyId = import_map.CurrencyId; price.ProductId = product.Id; price.Time = (DateTime)product.UpdateTime; decimal v; if (!decimal.TryParse(row[(int)import_map.C_PriceI].ToString().Trim(), out v)) { string error = "Cell (" + import_map.C_PriceI + ", " + row_number + ") Price cannot be parsed:" + row[(int)import_map.C_PriceI].ToString().Trim(); if (import.CheckNotImport) { invalid_price_count++; error_messages.Add(error); } else throw new Exception(error); } price.Value = v; if (!import.CheckNotImport) { Price p = db.Prices.Where(r => r.ProductId == price.ProductId && r.Time == price.Time).FirstOrDefault(); if (p == null) { db.Prices.Add(price); new_price_count += db.SaveChanges(); } else { price.Id = p.Id; db.Entry(p).CurrentValues.SetValues(price); updated_price_count += db.SaveChanges(); } } processed_price_count++; } DbApi.RenewContext(ref db); } } if (import.CheckNotImport) { Messages.Add("CHECK RESULT", "File " + import.File.FileName + " has been checked with map " + db.ImportMaps.Where(r => r.Id == import.MapId).First().Name); Messages.Add("CHECK RESULT", "Spreadsheets: " + sheet_count); Messages.Add("CHECK RESULT", "Found products: " + processed_product_count); Messages.Add("CHECK RESULT", "Invalid products: " + invalid_product_count); Messages.Add("CHECK RESULT", "Invalid prices: " + invalid_price_count); Messages.Add("CHECK RESULT", "Process time: " + String.Format("{0:0.00}", (DateTime.Now - start_time).TotalSeconds) + " secs"); add_error_messages("CHECK RESULT", error_messages); } else { Messages.Add("IMPORT RESULT", "File " + import.File.FileName + " has been imported with map " + db.ImportMaps.Where(r => r.Id == import.MapId).First().Name); Messages.Add("IMPORT RESULT", "Spreadsheets: " + sheet_count); Messages.Add("IMPORT RESULT", "Processed products: " + processed_product_count); Messages.Add("IMPORT RESULT", "Invalid products: " + invalid_product_count); Messages.Add("IMPORT RESULT", "Added products: " + new_product_count); Messages.Add("IMPORT RESULT", "Changed products: " + updated_product_count); Messages.Add("IMPORT RESULT", "Invalid prices: " + invalid_price_count); Messages.Add("IMPORT RESULT", "Added prices: " + new_price_count); Messages.Add("IMPORT RESULT", "Changed prices: " + updated_price_count); Messages.Add("CHECK RESULT", "Process time: " + String.Format("{0:0.00}", (DateTime.Now - start_time).TotalSeconds) + " secs"); add_error_messages("IMPORT RESULT", error_messages); } return true; }
bool process_file(string file, Import import, Action <int> progress = null) { DateTime start_time = DateTime.Now; Excel.IExcelDataReader edr; FileStream stream = System.IO.File.Open(file, FileMode.Open, FileAccess.Read); if (file.EndsWith(".xlsx", StringComparison.InvariantCultureIgnoreCase)) { edr = Excel.ExcelReaderFactory.CreateOpenXmlReader(stream); } else { edr = Excel.ExcelReaderFactory.CreateBinaryReader(stream); } //edr = Excel.ExcelReaderFactory.CreateOpenXmlReader(stream); System.Data.DataSet ds = edr.AsDataSet(); edr.Close(); Fhr.ProductOffice.Models.ImportMap import_map = db.ImportMaps.Where(r => r.Id == import.MapId).First(); //if (import_map.C_CompanyProductIdI == null) // throw new Exception("C_CompanyProductIdI in map #" + import_map.Id + " is not specified."); //if (import_map.C_NameI == null) // throw new Exception("C_NameI in map #" + import_map.Id + " is not specified."); db.Configuration.AutoDetectChangesEnabled = false; db.Configuration.ValidateOnSaveEnabled = false; int sheet_count = ds.Tables.Count; int invalid_product_count = 0; int new_product_count = 0; int updated_product_count = 0; int processed_product_count = 0; int invalid_price_count = 0; int new_price_count = 0; int updated_price_count = 0; int processed_price_count = 0; int row_number = -1; List <string> error_messages = new List <string>(); foreach (System.Data.DataTable dt in ds.Tables) { if (import_map.SkipFirstRow) { dt.Rows.RemoveAt(0); } foreach (System.Data.DataRow row in dt.Rows) { row_number++; Fhr.ProductOffice.Models.Product product = new Fhr.ProductOffice.Models.Product(); product.CompanyId = import_map.CompanyId; product.ExternalId = row[import_map.C_CompanyProductIdI].ToString().Trim(); if (string.IsNullOrWhiteSpace(product.ExternalId)) { bool line_is_empty = true; for (int i = 0; i < 5; i++) { if (!string.IsNullOrWhiteSpace(row[i].ToString())) { line_is_empty = false; break; } } if (!line_is_empty) { string error = "Cell (" + import_map.C_PriceI + ", " + row_number + ") ExternalId is empty."; if (import.CheckNotImport) { invalid_product_count++; error_messages.Add(error); } else { throw new Exception(error); } } continue; } product.ModifyTime = null; product.UpdateTime = import.UpdateTime; product.Name = row[import_map.C_NameI].ToString().Trim(); if (product.Name == null) { string error = "Cell (" + import_map.C_PriceI + ", " + row_number + ") Name is empty."; if (import.CheckNotImport) { invalid_product_count++; error_messages.Add(error); } else { throw new Exception(error); } } product.Source = "file:" + import.File.FileName; if (import_map.C_SkuI != null) { product.Sku = row[(int)import_map.C_SkuI].ToString().Trim(); } if (import_map.C_CategoryI != null) { product.Category = row[(int)import_map.C_CategoryI].ToString().Trim(); } if (import_map.C_DescriptionI != null) { product.Description = row[(int)import_map.C_DescriptionI].ToString().Trim(); } if (!import.CheckNotImport) { Fhr.ProductOffice.Models.Product p = db.Products.Where(r => r.CompanyId == product.CompanyId && r.ExternalId == product.ExternalId).FirstOrDefault(); if (p == null) { product.CreateTime = import.UpdateTime; product.LinkId = null; db.Products.Add(product); new_product_count += db.SaveChanges(); } else { product.Id = p.Id; if (product.Description != null) { p.Description = product.Description; } if (product.ImageUrls != null) { p.ImageUrls = product.ImageUrls; } if (product.ModifyTime != null) { p.ModifyTime = product.ModifyTime; } p.UpdateTime = product.UpdateTime; if (product.Name != null) { p.Name = product.Name; } if (product.Sku != null) { p.Sku = product.Sku; } if (product.Source != null) { p.Source = product.Source; } if (product.Url != null) { p.Url = product.Url; } //db.Entry(product).State = EntityState.Modified; updated_product_count += db.SaveChanges(); } } processed_product_count++; if (progress != null) { progress(processed_product_count); } if (import_map.C_PriceI != null) { Price price = new Price(); price.CurrencyId = import_map.CurrencyId; price.ProductId = product.Id; price.Time = (DateTime)product.UpdateTime; decimal v; if (!decimal.TryParse(row[(int)import_map.C_PriceI].ToString().Trim(), out v)) { string error = "Cell (" + import_map.C_PriceI + ", " + row_number + ") Price cannot be parsed:" + row[(int)import_map.C_PriceI].ToString().Trim(); if (import.CheckNotImport) { invalid_price_count++; error_messages.Add(error); } else { throw new Exception(error); } } price.Value = v; if (!import.CheckNotImport) { Price p = db.Prices.Where(r => r.ProductId == price.ProductId && r.Time == price.Time).FirstOrDefault(); if (p == null) { db.Prices.Add(price); new_price_count += db.SaveChanges(); } else { price.Id = p.Id; db.Entry(p).CurrentValues.SetValues(price); updated_price_count += db.SaveChanges(); } } processed_price_count++; } DbApi.RenewContext(ref db); } } if (import.CheckNotImport) { Messages.Add("CHECK RESULT", "File " + import.File.FileName + " has been checked with map " + db.ImportMaps.Where(r => r.Id == import.MapId).First().Name); Messages.Add("CHECK RESULT", "Spreadsheets: " + sheet_count); Messages.Add("CHECK RESULT", "Found products: " + processed_product_count); Messages.Add("CHECK RESULT", "Invalid products: " + invalid_product_count); Messages.Add("CHECK RESULT", "Invalid prices: " + invalid_price_count); Messages.Add("CHECK RESULT", "Process time: " + String.Format("{0:0.00}", (DateTime.Now - start_time).TotalSeconds) + " secs"); add_error_messages("CHECK RESULT", error_messages); } else { Messages.Add("IMPORT RESULT", "File " + import.File.FileName + " has been imported with map " + db.ImportMaps.Where(r => r.Id == import.MapId).First().Name); Messages.Add("IMPORT RESULT", "Spreadsheets: " + sheet_count); Messages.Add("IMPORT RESULT", "Processed products: " + processed_product_count); Messages.Add("IMPORT RESULT", "Invalid products: " + invalid_product_count); Messages.Add("IMPORT RESULT", "Added products: " + new_product_count); Messages.Add("IMPORT RESULT", "Changed products: " + updated_product_count); Messages.Add("IMPORT RESULT", "Invalid prices: " + invalid_price_count); Messages.Add("IMPORT RESULT", "Added prices: " + new_price_count); Messages.Add("IMPORT RESULT", "Changed prices: " + updated_price_count); Messages.Add("CHECK RESULT", "Process time: " + String.Format("{0:0.00}", (DateTime.Now - start_time).TotalSeconds) + " secs"); add_error_messages("IMPORT RESULT", error_messages); } return(true); }
void replicate_product_table(string crawler_id, string products_table) { Cliver.Fhr.ProductOffice.Models.DbApi.RenewContext(ref db2); try { Log.Write("Processing '" + products_table + "' table."); Fhr.ProductOffice.Models.Company company = (from r in db2.Companies where r.CrawlerId == crawler_id select r).FirstOrDefault(); if (company == null) { Log.Error("Could not find company for crawler_id:" + crawler_id); return; } int replicated_count = 0; Recordset products; do { Cliver.Fhr.ProductOffice.Models.DbApi.RenewContext(ref db2); products = Db["SELECT TOP 100 Id, CrawlTime, ChangeTime, Url, Data, State FROM " + products_table + " WHERE State=" + (int)Crawler.ProductState.NEW].GetRecordset(); foreach (Record record in products) { Fhr.CrawlerHost.Product raw_p = Cliver.CrawlerHost.Product.Restore <Fhr.CrawlerHost.Product>(record); Fhr.CrawlerHost.Product.PreparedProduct p = raw_p.GetPreparedProduct(); LogMessage.Write("Replicating id: '" + p.Id + "'"); Fhr.ProductOffice.Models.Product product = (from x in db2.Products where x.CompanyId == company.Id && x.ExternalId == p.Id select x).FirstOrDefault(); if (product != null) { product.Category = p.Category; product.Description = p.Description; product.ImageUrls = p.ImageUrls; product.ModifyTime = p.ChangeTime; product.Name = p.Name; product.Sku = p.Sku; product.Source = "crawler:" + crawler_id; product.Stock = p.Stock; product.UpdateTime = p.CrawlTime; product.Url = p.Url; } else { product = new Fhr.ProductOffice.Models.Product(); List <string> cs = new List <string>(); product.Category = p.Category; product.CompanyId = company.Id; product.ExternalId = p.Id; product.CreateTime = p.CrawlTime; product.Description = p.Description; product.ImageUrls = string.Join("\n", p.ImageUrls); product.LinkId = null; product.ModifyTime = p.ChangeTime; product.Name = p.Name; product.Sku = p.Sku; product.Source = "crawler:" + crawler_id; product.Stock = p.Stock; product.UpdateTime = p.CrawlTime; product.Url = p.Url; db2.Products.Add(product); } decimal price_value; Currency currency_id_; if (FhrDbReplicator.Parser.ParsePrice(p.Price, out currency_id_, out price_value)) { int currency_id = (int)currency_id_; Fhr.ProductOffice.Models.Price price = product.Prices.SingleOrDefault(x => x.Time == p.CrawlTime); //Price price = (from x in dc2.Prices where x.ProductId == product.Id && x.Time == (DateTime)r["CrawlTime"] && x.Quantity==1 select x).FirstOrDefault(); if (price != null) { price.CurrencyId = currency_id; price.Value = price_value; } else { price = new Fhr.ProductOffice.Models.Price(); price.CurrencyId = currency_id; price.ProductId = product.Id; price.Time = p.CrawlTime; price.Value = price_value; product.Prices.Add(price); } } db2.Configuration.ValidateOnSaveEnabled = false; db2.SaveChanges(); if (1 > Db["UPDATE " + products_table + " SET State=" + (int)Crawler.ProductState.REPLICATED + " WHERE Id=@Id"].Execute("@Id", p.Id)) { LogMessage.Error("Could not update State for product id:" + p.Id); } } replicated_count += products.Count; }while (products.Count > 0); LogMessage.Inform("Replicated products: " + replicated_count); int deleted_count = 0; do { Cliver.Fhr.ProductOffice.Models.DbApi.RenewContext(ref db2); products = Db["SELECT TOP 100 Id, CrawlTime, ChangeTime, Url, Data, State FROM " + products_table + " WHERE State=" + (int)Crawler.ProductState.DELETED].GetRecordset(); foreach (Record record in products) { LogMessage.Inform("Deleting id: '" + (string)record["Id"] + "'"); Fhr.ProductOffice.Models.Product product = (from x in db2.Products where x.CompanyId == company.Id && x.ExternalId == (string)record["Id"] select x).FirstOrDefault(); if (product == null) { Log.Warning("Could not find Product [CompanyId=" + company.Id + "CompanyProductId=" + (string)record["Id"] + "] while deleting."); } else { Cliver.Fhr.ProductOffice.DataApi.Product.Delete(db2, product.Id); int u = Db["DELETE FROM " + products_table + " WHERE Id=@Id"].Execute("@Id", (string)record["Id"]); if (u < 1) { LogMessage.Error("Could not delete record for product id:" + (string)record["Id"]); } } } deleted_count += products.Count; }while (products.Count > 0); LogMessage.Inform("Deleted products: " + deleted_count); } catch (Exception e) { LogMessage.Error(e); } }
void replicate_product_table(string crawler_id, string products_table) { Cliver.Fhr.ProductOffice.Models.DbApi.RenewContext(ref db2); try { Log.Write("Processing '" + products_table + "' table."); Fhr.ProductOffice.Models.Company company = (from r in db2.Companies where r.CrawlerId == crawler_id select r).FirstOrDefault(); if (company == null) { Log.Error("Could not find company for crawler_id:" + crawler_id); return; } int replicated_count = 0; Recordset products; do { Cliver.Fhr.ProductOffice.Models.DbApi.RenewContext(ref db2); products = Db["SELECT TOP 100 Id, CrawlTime, ChangeTime, Url, Data, State FROM " + products_table + " WHERE State=" + (int)Crawler.ProductState.NEW].GetRecordset(); foreach (Record record in products) { Fhr.CrawlerHost.Product raw_p = Cliver.CrawlerHost.Product.Restore<Fhr.CrawlerHost.Product>(record); Fhr.CrawlerHost.Product.PreparedProduct p = raw_p.GetPreparedProduct(); LogMessage.Write("Replicating id: '" + p.Id + "'"); Fhr.ProductOffice.Models.Product product = (from x in db2.Products where x.CompanyId == company.Id && x.ExternalId == p.Id select x).FirstOrDefault(); if (product != null) { product.Category = p.Category; product.Description = p.Description; product.ImageUrls = p.ImageUrls; product.ModifyTime = p.ChangeTime; product.Name = p.Name; product.Sku = p.Sku; product.Source = "crawler:" + crawler_id; product.Stock = p.Stock; product.UpdateTime = p.CrawlTime; product.Url = p.Url; } else { product = new Fhr.ProductOffice.Models.Product(); List<string> cs = new List<string>(); product.Category = p.Category; product.CompanyId = company.Id; product.ExternalId = p.Id; product.CreateTime = p.CrawlTime; product.Description = p.Description; product.ImageUrls = string.Join("\n", p.ImageUrls); product.LinkId = null; product.ModifyTime = p.ChangeTime; product.Name = p.Name; product.Sku = p.Sku; product.Source = "crawler:" + crawler_id; product.Stock = p.Stock; product.UpdateTime = p.CrawlTime; product.Url = p.Url; db2.Products.Add(product); } decimal price_value; Currency currency_id_; if (FhrDbReplicator.Parser.ParsePrice(p.Price, out currency_id_, out price_value)) { int currency_id = (int)currency_id_; Fhr.ProductOffice.Models.Price price = product.Prices.SingleOrDefault(x => x.Time == p.CrawlTime); //Price price = (from x in dc2.Prices where x.ProductId == product.Id && x.Time == (DateTime)r["CrawlTime"] && x.Quantity==1 select x).FirstOrDefault(); if (price != null) { price.CurrencyId = currency_id; price.Value = price_value; } else { price = new Fhr.ProductOffice.Models.Price(); price.CurrencyId = currency_id; price.ProductId = product.Id; price.Time = p.CrawlTime; price.Value = price_value; product.Prices.Add(price); } } db2.Configuration.ValidateOnSaveEnabled = false; db2.SaveChanges(); if (1 > Db["UPDATE " + products_table + " SET State=" + (int)Crawler.ProductState.REPLICATED + " WHERE Id=@Id"].Execute("@Id", p.Id)) LogMessage.Error("Could not update State for product id:" + p.Id); } replicated_count += products.Count; } while (products.Count > 0); LogMessage.Inform("Replicated products: " + replicated_count); int deleted_count = 0; do { Cliver.Fhr.ProductOffice.Models.DbApi.RenewContext(ref db2); products = Db["SELECT TOP 100 Id, CrawlTime, ChangeTime, Url, Data, State FROM " + products_table + " WHERE State=" + (int)Crawler.ProductState.DELETED].GetRecordset(); foreach (Record record in products) { LogMessage.Inform("Deleting id: '" + (string)record["Id"] + "'"); Fhr.ProductOffice.Models.Product product = (from x in db2.Products where x.CompanyId == company.Id && x.ExternalId == (string)record["Id"] select x).FirstOrDefault(); if (product == null) Log.Warning("Could not find Product [CompanyId=" + company.Id + "CompanyProductId=" + (string)record["Id"] + "] while deleting."); else { Cliver.Fhr.ProductOffice.DataApi.Product.Delete(db2, product.Id); int u = Db["DELETE FROM " + products_table + " WHERE Id=@Id"].Execute("@Id", (string)record["Id"]); if (u < 1) LogMessage.Error("Could not delete record for product id:" + (string)record["Id"]); } } deleted_count += products.Count; } while (products.Count > 0); LogMessage.Inform("Deleted products: " + deleted_count); } catch (Exception e) { LogMessage.Error(e); } }