public static void GetProductInfo() { XymogenScraper scraper = new XymogenScraper(); Dictionary <long, XymogenProduct> prods = new Dictionary <long, XymogenProduct>(); if (File.Exists(path)) { prods = JsonConvert.DeserializeObject <Dictionary <long, XymogenProduct> >(File.ReadAllText(path)); } using (WebClient wc = new WebClient()) { for (int id = startingId; id < endingId; id++) { System.Console.Write($"{id}\r"); if (!prods.Any(p => p.Value.Variants.Any(v => v.ProductID == id))) { XymogenProduct prod = scraper.ProcessPage(id); if (prod != null) { if (!prods.ContainsKey(prod.MasterId)) { prods.Add(prod.MasterId, prod); string s = JsonConvert.SerializeObject(prods); File.WriteAllText(path, s); } else { } } } } } }
public XymogenProduct processNode(HtmlNode node) { XymogenProduct product = new XymogenProduct(); HtmlNode nameNode = node.SelectSingleNode(".//h2"); HtmlNode miniNode = node.SelectSingleNode(".//h4[@class='pi-text-base pi-has-border pi-visible-xs']"); HtmlNode categoryNode = node.SelectSingleNode(".//ul[@class='pi-meta']"); HtmlNode briefNode = node.SelectSingleNode(".//ul[@class='pi-list-with-icons pi-list-icons-right-open']"); HtmlNode descNode = node.SelectSingleNode(".//h4[contains(., 'Formula Details')]").NextSibling; product.Name = nameNode.InnerText.Trim(); product.Benefits = briefNode?.InnerHtml; if (categoryNode != null) { string category = categoryNode?.InnerText.Trim(); product.Categories = category.Split(new string[] { ", " }, StringSplitOptions.RemoveEmptyEntries).ToList(); } product.BriefDescription = miniNode.InnerText.Trim(); product.Description = descNode.InnerHtml.Trim(); HtmlNode directNode = node.SelectSingleNode(".//a[contains(., 'Directions:')]/../../div"); product.Directions = directNode?.InnerText.Trim(); HtmlNode containsNode = node.SelectSingleNode(".//a[contains(., 'Contains:')]/../../div"); product.Contains = containsNode?.InnerText.Trim(); HtmlNode dncNode = node.SelectSingleNode(".//a[contains(., 'Does Not Contain:')]/../../div"); product.DoesNotContain = dncNode?.InnerText.Trim(); HtmlNode ingNode = node.SelectSingleNode(".//a[contains(., 'Other Ingredients:')]/../../div"); product.Ingredients = ingNode?.InnerText.Trim(); HtmlNode storageNode = node.SelectSingleNode(".//a[contains(., 'Storage:')]/../../div"); product.Storage = storageNode?.InnerText.Trim(); HtmlNode patentNode = node.SelectSingleNode(".//a[contains(., 'Patent Info:')]/../../div"); product.PatentInfo = patentNode?.InnerText.Trim(); string imageNode = node.SelectSingleNode(".//img").Attributes["src"].Value.Split(new char[] { '=', '&' })[1]; product.Id = Convert.ToInt64(imageNode); //https://www.xymogen.com/assets/imageDisplay.ashx?productID=92&attachmentTypeID=2 return(product); }
private void GetOptions(XymogenProduct p) { using (WebClient wc = new WebClient()) { wc.Headers[HttpRequestHeader.ContentType] = "application/json"; string response = wc.UploadString("https://www.xymogen.com/usercontrols/utilities/ProductVariants.asmx/LoadProductVariantOptions", $"{{MasterId: {p.MasterId}, variantSelections: null, ProductID: 0, Favorites: false}}"); Wrapper o = JsonConvert.DeserializeObject <Wrapper>(response); string[] parts = o.d.Substring(1, o.d.Length - 2).Split("],"); p.Options = JsonConvert.DeserializeObject <List <XymogenProduct.Option> >(parts[0].Trim() + "]"); int index = 1; foreach (XymogenProduct.Option t in p.Options) { string s = parts[index].Trim() + "]"; t.values = JsonConvert.DeserializeObject <List <XymogenProduct.Value> >(s); index++; } List <combos> vars = buildVariants(p.Options, 0); foreach (combos variant in vars) { wc.Headers[HttpRequestHeader.ContentType] = "application/json"; response = wc.UploadString("https://www.xymogen.com/usercontrols/utilities/ProductVariants.asmx/LoadProductVariantOptions", $"{{MasterId: {p.MasterId}, variantSelections: [{string.Join(",", variant.ids)}], ProductID: 0, Favorites: false}}"); o = JsonConvert.DeserializeObject <Wrapper>(response); if (!string.IsNullOrEmpty(o.d)) { parts = o.d.Substring(1, o.d.Length - 2).Split("],"); XymogenProduct.Variant ppp = JsonConvert.DeserializeObject <List <XymogenProduct.Variant> >(parts.Last())[0]; ppp.Option1 = variant.values[0]; if (variant.values.Count > 1) { ppp.Option2 = variant.values[1]; } if (variant.values.Count > 2) { ppp.Option3 = variant.values[2]; } ppp.SKU = $"XMOGEN-{ppp.ProductID}"; ppp.BarCode = GetUPCCodes(ppp.ProductID); p.Variants.Add(ppp); } } } }
public XymogenProduct ProcessPage(int id, WebClient wc = null) { string url = $"https://www.xymogen.com/formulas/products/{id}"; HtmlDocument doc = GetPage(url, wc); //WebClient client = wc; //if (client == null) //{ // client = new WebClient(); //} //rawHtml = client.DownloadString(new Uri(url)); //if (wc == null) //{ // client.Dispose(); //} //while (rawHtml.Contains("\n ")) //{ // rawHtml = rawHtml.Replace("\n ", "\n"); //} //rawHtml = rawHtml.Replace("\n", "\r\n"); //rawHtml = rawHtml.Replace("\r\r\n", "\r\n"); //rawHtml = rawHtml.Replace("\r\n\r\n", "\r\n"); //rawHtml = rawHtml.Replace("\r\n", "\n"); //HtmlWeb web = new HtmlWeb(); //HtmlDocument doc = new HtmlDocument(); //doc.LoadHtml(rawHtml); //sanitizeNode(doc.DocumentNode); HtmlNodeCollection n2 = doc.DocumentNode.SelectNodes("//div[@id='udpProductDetails']/div/div"); if (n2 != null) { foreach (HtmlNode node in n2) { XymogenProduct p = processNode(node); if (p.Id == id) { p.MasterId = findMasterId(id); GetOptions(p); return(p); } } } return(null); }
private static async Task Supplement2Async() { List <InvSupplement> inv2 = JsonConvert.DeserializeObject <List <InvSupplement> >(File.ReadAllText(@"C:\Users\Robert\Documents\supplements.json")); List <InvSupplement> inv = inv2.Where(i => i.Company == "Xymogen" && i.Active && !i.Imported && i.XymogenId != 0).ToList(); Dictionary <long, XymogenScraper.XymogenProduct> prods = GetCacheProductInfo(); Dictionary <long, List <int> > toImport = new Dictionary <long, List <int> >(); foreach (InvSupplement i in inv) { List <XymogenScraper.XymogenProduct> p = prods.Values.Where(xp => xp.Variants.Any(xv => xv.ProductID == i.XymogenId)).ToList(); if (p.Count == 1) { if (toImport.ContainsKey(p[0].MasterId)) { if (!toImport[p[0].MasterId].Contains(i.XymogenId)) { toImport[p[0].MasterId].Add(i.XymogenId); } else { System.Console.WriteLine($"{i.Name} - {i.XymogenId} - {i.Size} - {i.Flavor}"); } } else { toImport.Add(p[0].MasterId, new List <int>() { i.XymogenId }); } } else { System.Console.WriteLine($"{i.Name} - {i.XymogenId} - {i.Size} - {i.Flavor}"); } } foreach (KeyValuePair <long, List <int> > vp in toImport) { XymogenProduct prod = prods.Values.FirstOrDefault(p => p.MasterId == vp.Key); if (prod != null) { prod.Variants.RemoveAll(v => !vp.Value.Contains(v.ProductID)); foreach (InvSupplement i in inv2.Where(iv => vp.Value.Contains(iv.XymogenId))) { i.Imported = true; XymogenProduct.Variant v = prod.Variants.FirstOrDefault(pv => pv.ProductID == i.XymogenId); if (v != null) { if (i.Retail == 0.00m) { } v.Price = i.Retail; } else { } } System.Console.Write($"{prod.Name} "); await BuildProductAsync(prod); File.WriteAllText(@"C:\Users\Robert\Documents\supplements.json", JsonConvert.SerializeObject(inv2, Formatting.Indented, new JsonSerializerSettings { NullValueHandling = NullValueHandling.Ignore })); System.Console.WriteLine($"completed"); } else { } } }
private static async Task BuildProductAsync(int id) { XymogenProduct p = GetCacheProductInfo(id); await BuildProductAsync(p); }
private static async Task BuildProductAsync(XymogenProduct p) { Product prod = new Product { Options = new List <ProductOption>(), Variants = new List <ProductVariant>(), Images = new List <ProductImage>(), PublishedScope = "global", Title = p.Name, BodyHtml = p.Description, Vendor = "Xymogen" }; foreach (XymogenProduct.Option vt in p.Options) { ProductOption po = new ProductOption { Values = new List <string>(), Name = vt.variantTypeName, ProductId = prod.Id }; foreach (XymogenProduct.Value vv in vt.values) { (po.Values as List <string>).Add(vv.variantTypeValueName); } (prod.Options as List <ProductOption>).Add(po); } foreach (XymogenProduct.Variant v in p.Variants) { ProductVariant pv = new ProductVariant { Barcode = v.BarCode, Option1 = v.Option1, Option2 = v.Option2, Option3 = v.Option3, SKU = v.SKU, Taxable = true, InventoryManagement = "shopify", InventoryQuantity = 0, Price = v.Price }; (prod.Variants as List <ProductVariant>).Add(pv); ProductImage image = new ProductImage { Attachment = Convert.ToBase64String(File.ReadAllBytes($"f:\\temp\\Images\\{v.ProductID}.png")), Filename = $"{v.ProductID}.png" }; (prod.Images as List <ProductImage>).Add(image); } prod = await Shopify.CreateProductAsync(prod); foreach (ProductVariant pv in prod.Variants) { string vid = pv.SKU.Split('-')[1]; ProductImage pi = prod.Images.FirstOrDefault(pii => pii.Src.Contains($"/{vid}.png")); List <long> ids = new List <long> { pv.Id.Value }; pi.VariantIds = ids; } prod = await Shopify.UpdateProductAsync(prod); Shopify.CreateProductMetadata(prod.Id.Value, "ownProduct", "ShortDescription", p.BriefDescription); Shopify.CreateProductMetadata(prod.Id.Value, "ownProduct", "Benefits", p.Benefits); Shopify.CreateProductMetadata(prod.Id.Value, "ownProduct", "Contains", p.Contains); Shopify.CreateProductMetadata(prod.Id.Value, "ownProduct", "Directions", p.Directions); Shopify.CreateProductMetadata(prod.Id.Value, "ownProduct", "DoesNotContain", p.DoesNotContain); Shopify.CreateProductMetadata(prod.Id.Value, "ownProduct", "Ingredients", p.Ingredients); Shopify.CreateProductMetadata(prod.Id.Value, "ownProduct", "PatentInfo", p.PatentInfo); Shopify.CreateProductMetadata(prod.Id.Value, "ownProduct", "Storage", p.Storage); foreach (string collect in p.Categories) { Shopify.AddProductToCollection(prod, collect); } }
private static void Supplement() { Dictionary <long, XymogenProduct> prods = GetCacheProductInfo(); List <string> names = new List <string>(); foreach (XymogenProduct prod in prods.Values) { names.Add(prod.Name); prod.Name = prod.Name.Replace("™", "").Replace("®", ""); } names.Sort(); List <InvSupplement> inv = JsonConvert.DeserializeObject <List <InvSupplement> >(File.ReadAllText(@"C:\Users\Robert\Documents\supplements.json")); foreach (InvSupplement i in inv) { i.XymogenId = 0; if (i.Company == "Xymogen" && i.Active) { //string[] s = i.Supplement.Split(" - "); //i.Name = s[0]; //if (s.Length >= 3) //{ // i.Dosage = s[1]; // i.Size = s[2]; //} //if (s.Length == 2) //{ // i.Size = s[1]; //} if (i.Size != null) { // i.Size = i.Size.Replace("caps", "C").Replace("tabs", "T").Replace("serv", "Serv").Replace("pkts", "Serv").Replace("oz", " oz."); if (i.Active) { XymogenProduct prod = prods.Values.FirstOrDefault(p => p.Name == i.Name); if (prod == null) { System.Console.WriteLine($"No Matching Xymogen Product - {i.Name}"); } else { XymogenProduct.Variant v; if (i.Flavor == null) { XymogenProduct.Option op = prod.Options.FirstOrDefault(o => o.variantTypeName == "Flavor"); if (op != null && op.values.Count == 1) { i.Flavor = op.values[0].variantTypeValueName; } } if (i.Flavor == null) { v = prod.Variants.FirstOrDefault(xv => xv.Option1 == i.Size && xv.Option2 == null); } else { v = prod.Variants.FirstOrDefault(xv => (xv.Option1 == i.Size || xv.Option2 == i.Size) && (xv.Option1 == i.Flavor || xv.Option2 == i.Flavor)); } if (v == null) { System.Console.WriteLine($"No Matching Xymogen Variant - {i.Name} - {i.Size} - {i.Flavor}"); } else { i.XymogenId = v.ProductID; } } } } else { System.Console.WriteLine($"No Size Xymogen Product - {i.Name}"); } } } File.WriteAllText(@"C:\Users\Robert\Documents\supplements.json", JsonConvert.SerializeObject(inv, Formatting.Indented, new JsonSerializerSettings { NullValueHandling = NullValueHandling.Ignore })); }