private ShopifyData ReadData(ShopifyImportParams importParams, ShopifyImportNotification notification) { var result = new ShopifyData(); if (importParams.ImportProducts) { ReadProducts(notification, result); } if (importParams.ImportCollections) { ReadCollections(notification, result); } if (importParams.ImportThemes) { ReadThemes(notification, result); } if (importParams.ImportPages) { ReadPages(notification, result); } //TODO read another data return(result); }
public IHttpActionResult StartImport(ShopifyImportParams importParams) { var notification = new ShopifyImportNotification(CurrentPrincipal.GetCurrentUserName()) { Title = "Import catalog from Shopify", Description = "starting import...." }; _notifier.Upsert(notification); BackgroundJob.Enqueue(() => _shopifyImportService.Import(importParams, notification)); return(Ok(notification)); }
private void ReadCollections(ShopifyImportNotification notification, ShopifyData result) { notification.Description = "Reading collects from shopify..."; _notifier.Upsert(notification); result.Collects = _shopifyRepository.GetShopifyCollects(); notification.Description = "Reading collections from shopify..."; _notifier.Upsert(notification); result.Collections = _shopifyRepository.GetShopifyCollections(); notification.Progresses.Add("Collections", new ShopifyImportProgress() { TotalCount = result.Collections.Count() }); }
private void UpdateProgresses(ShopifyImportParams importParams, ShopifyImportNotification notification, ShopifyData shopifyData) { notification.Progresses.Clear(); if (importParams.ImportProducts) { notification.Progresses.Add(ProductsKey, new ShopifyImportProgress() { TotalCount = shopifyData.Products.Count() }); notification.Progresses.Add(PropertiesKey, new ShopifyImportProgress() { TotalCount = shopifyData.Products.SelectMany(product => product.Options).GroupBy(o => o.Name).Count() }); } if (importParams.ImportCollections) { notification.Progresses.Add(CollectionsKey, new ShopifyImportProgress() { TotalCount = shopifyData.Collections.Count() }); } if (importParams.ImportThemes) { notification.Progresses.Add(ThemesKey, new ShopifyImportProgress() { TotalCount = shopifyData.Themes.Count() }); } if (importParams.ImportPages) { notification.Progresses.Add(PagesKey, new ShopifyImportProgress() { TotalCount = shopifyData.Pages.Count() }); } _notifier.Upsert(notification); }
public ShopifyImportNotification Import(ShopifyImportParams importParams, ShopifyImportNotification notification) { try { var shopifyData = ReadData(importParams, notification); UpdateProgresses(importParams, notification, shopifyData); var virtoData = ConvertData(shopifyData, importParams, notification); SaveData(virtoData, importParams, notification); } catch (Exception ex) { notification.Description = "Import error"; notification.ErrorCount++; notification.Errors.Add(ex.ToString()); } finally { notification.Finished = DateTime.UtcNow; notification.Description = "Import finished" + (notification.Errors.Any() ? " with errors" : " successfully"); _notifier.Upsert(notification); } return(notification); }
private void SaveData(VirtoData virtoData, ShopifyImportParams importParams, ShopifyImportNotification notification) { if (importParams.ImportCollections) { SaveCategories(virtoData, importParams, notification); } if (importParams.ImportProducts) { SaveProperties(virtoData, importParams, notification); SaveProducts(virtoData, importParams, notification); } if (importParams.ImportThemes) { SaveThemes(virtoData, importParams, notification); } if (importParams.ImportPages) { SavePages(virtoData, importParams, notification); } }
private void ReadProducts(ShopifyImportNotification notification, ShopifyData result) { notification.Description = "Reading products from shopify..."; _notifier.Upsert(notification); result.Products = _shopifyRepository.GetShopifyProducts(); }
private void ReadThemes(ShopifyImportNotification notification, ShopifyData result) { result.Themes = new Dictionary <ShopifyTheme, Stream>(); notification.Description = "Reading themes from shopify..."; var themes = _shopifyRepository.GetShopifyThemes().ToList(); notification.Progresses.Clear(); _notifier.Upsert(notification); foreach (var theme in themes) { var assets = _shopifyRepository.GetShopifyAssets(theme.Id).ToList(); var themeProgress = new ShopifyImportProgress() { TotalCount = assets.Count }; notification.Progresses.Add(string.Format("{0} theme assets downloading ", theme.Name), themeProgress); _notifier.Upsert(notification); var stream = new MemoryStream(); using (var zip = new ZipArchive(stream, ZipArchiveMode.Create, true)) { foreach (var asset in assets) { try { var downloadedAsset = _shopifyRepository.DownloadShopifyAsset(theme.Id, asset); var entry = zip.CreateEntry(string.Format("{0}/{1}", theme.Id, asset.Key)); using (var entryStream = entry.Open()) { if (downloadedAsset.Value != null) { using (var writer = new StreamWriter(entryStream)) { writer.Write(downloadedAsset.Value); } } else { if (downloadedAsset.Attachment != null) { var data = Convert.FromBase64String(downloadedAsset.Attachment); entryStream.Write(data, 0, data.Length); } } } } catch (Exception ex) { themeProgress.ErrorCount++; notification.ErrorCount++; notification.Errors.Add(ex.ToString()); _notifier.Upsert(notification); } finally { //Raise notification each notifyProductSizeLimit category themeProgress.ProcessedCount++; if (themeProgress.ProcessedCount % NotifySizeLimit == 0 || themeProgress.ProcessedCount + themeProgress.ErrorCount == themeProgress.TotalCount) { _notifier.Upsert(notification); } } } } stream.Seek(0, SeekOrigin.Begin); result.Themes.Add(theme, stream); } }
private VirtoData ConvertData(ShopifyData shopifyData, ShopifyImportParams importParams, ShopifyImportNotification notification) { var virtoData = new VirtoData(); if (importParams.ImportCollections) { notification.Description = "Converting categories"; _notifier.Upsert(notification); virtoData.Categories = shopifyData.Collections .Select(collection => _shopifyConverter.Convert(collection, importParams)).ToList(); } if (importParams.ImportProducts) { notification.Description = "Converting product properties"; _notifier.Upsert(notification); virtoData.Properties = shopifyData.Products.SelectMany(product => product.Options) .Select(option => _shopifyConverter.Convert(option, importParams)) .ToList(); notification.Description = "Converting products"; _notifier.Upsert(notification); virtoData.Products = shopifyData.Products.Select( product => _shopifyConverter.Convert(product, importParams, shopifyData, virtoData)).ToList(); } if (importParams.ImportThemes) { virtoData.Themes = shopifyData.Themes; } if (importParams.ImportPages) { virtoData.Pages = shopifyData.Pages.Select(page => _shopifyConverter.Convert(page)).ToList(); } return(virtoData); }
private void SaveThemes(VirtoData virtoData, ShopifyImportParams importParams, ShopifyImportNotification notification) { notification.Description = "Saving themes"; _notifier.Upsert(notification); foreach (var theme in virtoData.Themes) { using (ZipArchive archive = new ZipArchive(theme.Value, ZipArchiveMode.Read)) { foreach (ZipArchiveEntry entry in archive.Entries) { if (!entry.FullName.EndsWith("/")) { var fileName = String.Join("/", entry.FullName.Split('/').Skip(1)); using (var entryStream = entry.Open()) using (var targetStream = _contentStorageProvider.OpenWrite("/Themes/" + importParams.StoreId + "/" + theme.Key.Name + "/" + fileName)) { entryStream.CopyTo(targetStream); } } } } notification.Progresses[ThemesKey].ProcessedCount++; _notifier.Upsert(notification); } }
private void SaveProducts(VirtoData virtoData, ShopifyImportParams importParams, ShopifyImportNotification notification) { var productProgress = notification.Progresses[ProductsKey]; if (importParams.ImportCollections) { //Update categories references foreach (var product in virtoData.Products) { if (product.Category != null) { product.Category = virtoData.Categories.First(category => category.Code == product.Category.Code); product.CategoryId = product.Category.Id; } } } notification.Description = "Checking existing products"; _notifier.Upsert(notification); notification.Description = "Saving products"; _notifier.Upsert(notification); var productsToCreate = new List <coreModel.CatalogProduct>(); var productsToUpdate = new List <coreModel.CatalogProduct>(); foreach (var product in virtoData.Products) { var existingProduct = _searchService.Search(new coreModel.SearchCriteria() { CatalogId = importParams.VirtoCatalogId, GetAllCategories = true, ResponseGroup = coreModel.ResponseGroup.WithProducts, Count = int.MaxValue, Code = product.Code }).Products.FirstOrDefault(); if (existingProduct != null) { product.Id = existingProduct.Id; productsToUpdate.Add(product); } else { productsToCreate.Add(product); } } foreach (var product in productsToCreate) { try { _productService.Create(product); //Create price in default price list if (product.Prices != null && product.Prices.Any()) { var price = product.Prices.First(); price.ProductId = product.Id; _pricingService.CreatePrice(price); } } catch (Exception ex) { productProgress.ErrorCount++; notification.ErrorCount++; notification.Errors.Add(ex.ToString()); _notifier.Upsert(notification); } finally { //Raise notification each notifyProductSizeLimit category productProgress.ProcessedCount++; notification.Description = string.Format("Creating products: {0} of {1} created", productProgress.ProcessedCount, productProgress.TotalCount); if (productProgress.ProcessedCount % NotifySizeLimit == 0 || productProgress.ProcessedCount + productProgress.ErrorCount == productProgress.TotalCount) { _notifier.Upsert(notification); } } } if (productsToUpdate.Count > 0) { try { _productService.Update(productsToUpdate.ToArray()); } catch (Exception ex) { productProgress.ErrorCount++; notification.ErrorCount++; notification.Errors.Add(ex.ToString()); _notifier.Upsert(notification); } finally { notification.Description = string.Format("Updating products: {0} updated", productProgress.ProcessedCount = productProgress.ProcessedCount + productsToUpdate.Count); _notifier.Upsert(notification); } } virtoData.Products.Clear(); virtoData.Products.AddRange(productsToCreate); virtoData.Products.AddRange(productsToUpdate); }
private void SaveThemes(VirtoData virtoData, ShopifyImportParams importParams, ShopifyImportNotification notification) { notification.Description = "Saving themes"; _notifier.Upsert(notification); foreach (var theme in virtoData.Themes) { using (var zip = new ZipArchive(theme.Value, ZipArchiveMode.Read)) { _themeService.UploadTheme(importParams.StoreId, theme.Key.Name, zip); } notification.Progresses[ThemesKey].ProcessedCount++; _notifier.Upsert(notification); } }
private void SaveProperties(VirtoData virtoData, ShopifyImportParams importParams, ShopifyImportNotification notification) { var propertiesProgress = notification.Progresses[PropertiesKey]; notification.Description = "Saving properties"; _notifier.Upsert(notification); var propertiesToCreate = new List <coreModel.Property>(); var propertiesToUpdate = new List <coreModel.Property>(); var existingProperties = _propertyService.GetCatalogProperties(importParams.VirtoCatalogId); foreach (var property in virtoData.Properties) { if (propertiesToCreate.All(p => p.Name != property.Name) && propertiesToUpdate.All(p => p.Name != property.Name)) { var existitngProperty = existingProperties.FirstOrDefault(c => c.Name == property.Name); if (existitngProperty != null) { property.Id = existitngProperty.Id; propertiesToUpdate.Add(property); } else { propertiesToCreate.Add(property); } } } foreach (var property in propertiesToCreate) { try { _propertyService.Create(property); } catch (Exception ex) { propertiesProgress.ErrorCount++; notification.ErrorCount++; notification.Errors.Add(ex.ToString()); _notifier.Upsert(notification); } finally { //Raise notification each notifyProductSizeLimit property propertiesProgress.ProcessedCount++; notification.Description = string.Format("Creating properties: {0} of {1} created", propertiesProgress.ProcessedCount, propertiesProgress.TotalCount); if (propertiesProgress.ProcessedCount % NotifySizeLimit == 0 || propertiesProgress.ProcessedCount + propertiesProgress.ErrorCount == propertiesProgress.TotalCount) { _notifier.Upsert(notification); } } } if (propertiesToUpdate.Count > 0) { _propertyService.Update(propertiesToUpdate.ToArray()); notification.Description = string.Format("Updating properties: {0} updated", propertiesProgress.ProcessedCount = propertiesProgress.ProcessedCount + propertiesToUpdate.Count); _notifier.Upsert(notification); } virtoData.Properties.Clear(); virtoData.Properties.AddRange(propertiesToCreate); virtoData.Properties.AddRange(propertiesToUpdate); }
private void SavePages(VirtoData virtoData, ShopifyImportParams importParams, ShopifyImportNotification notification) { notification.Description = "Saving pages"; _notifier.Upsert(notification); foreach (var page in virtoData.Pages) { _pagesService.SavePage(importParams.StoreId, page); notification.Progresses[PagesKey].ProcessedCount++; _notifier.Upsert(notification); } }
private void SaveProducts(VirtoData virtoData, ShopifyImportParams importParams, ShopifyImportNotification notification) { var productProgress = notification.Progresses[ProductsKey]; if (importParams.ImportCollections) { //Update categories references foreach (var product in virtoData.Products) { if (product.Category != null) { product.Category = virtoData.Categories.First(category => category.Code == product.Category.Code); product.CategoryId = product.Category.Id; } } } notification.Description = "Checking existing products"; _notifier.Upsert(notification); notification.Description = "Saving products"; _notifier.Upsert(notification); var productsToCreate = new List <coreModel.CatalogProduct>(); var productsToUpdate = new List <coreModel.CatalogProduct>(); var alreadyExistProducts = _productService.GetByIds(virtoData.Products.Select(x => x.Id).Where(x => x != null).ToArray(), coreModel.ItemResponseGroup.ItemInfo); foreach (var product in virtoData.Products) { if (alreadyExistProducts.Any(x => x.Id == product.Id)) { productsToUpdate.Add(product); } else { productsToCreate.Add(product); } } foreach (var product in productsToCreate) { try { _productService.Create(product); //Create price in default price list if (product.Prices != null && product.Prices.Any()) { var price = product.Prices.First(); price.ProductId = product.Id; _pricingService.CreatePrice(price); } } catch (Exception ex) { productProgress.ErrorCount++; notification.ErrorCount++; notification.Errors.Add(ex.ToString()); _notifier.Upsert(notification); } finally { //Raise notification each notifyProductSizeLimit category productProgress.ProcessedCount++; notification.Description = string.Format("Creating products: {0} of {1} created", productProgress.ProcessedCount, productProgress.TotalCount); if (productProgress.ProcessedCount % NotifySizeLimit == 0 || productProgress.ProcessedCount + productProgress.ErrorCount == productProgress.TotalCount) { _notifier.Upsert(notification); } } } if (productsToUpdate.Count > 0) { try { _productService.Update(productsToUpdate.ToArray()); } catch (Exception ex) { productProgress.ErrorCount++; notification.ErrorCount++; notification.Errors.Add(ex.ToString()); _notifier.Upsert(notification); } finally { notification.Description = string.Format("Updating products: {0} updated", productProgress.ProcessedCount = productProgress.ProcessedCount + productsToUpdate.Count); _notifier.Upsert(notification); } } virtoData.Products.Clear(); virtoData.Products.AddRange(productsToCreate); virtoData.Products.AddRange(productsToUpdate); }
private void SaveCategories(VirtoData virtoData, ShopifyImportParams importParams, ShopifyImportNotification notification) { var categoriesProgress = notification.Progresses[CollectionsKey]; notification.Description = "Saving categories"; _notifier.Upsert(notification); var categoriesToCreate = new List <coreModel.Category>(); var categoriesToUpdate = new List <coreModel.Category>(); var existingCategories = _searchService.Search(new coreModel.SearchCriteria() { CatalogId = importParams.VirtoCatalogId, GetAllCategories = true, ResponseGroup = coreModel.ResponseGroup.WithCategories }).Categories; foreach (var category in virtoData.Categories) { var existingCategory = existingCategories.FirstOrDefault(c => c.Code == category.Code); if (existingCategory != null) { category.Id = existingCategory.Id; categoriesToUpdate.Add(category); } else { categoriesToCreate.Add(category); } } foreach (var category in categoriesToCreate) { try { _categoryService.Create(category); } catch (Exception ex) { categoriesProgress.ErrorCount++; notification.ErrorCount++; notification.Errors.Add(ex.ToString()); _notifier.Upsert(notification); } finally { //Raise notification each notifyProductSizeLimit category categoriesProgress.ProcessedCount++; notification.Description = string.Format("Creating categories: {0} of {1} created", categoriesProgress.ProcessedCount, categoriesProgress.TotalCount); if (categoriesProgress.ProcessedCount % NotifySizeLimit == 0 || categoriesProgress.ProcessedCount + categoriesProgress.ErrorCount == categoriesProgress.TotalCount) { _notifier.Upsert(notification); } } } if (categoriesToUpdate.Count > 0) { _categoryService.Update(categoriesToUpdate.ToArray()); notification.Description = string.Format("Updating categories: {0} updated", categoriesProgress.ProcessedCount = categoriesProgress.ProcessedCount + categoriesToUpdate.Count); _notifier.Upsert(notification); } virtoData.Categories.Clear(); virtoData.Categories.AddRange(categoriesToCreate); virtoData.Categories.AddRange(categoriesToUpdate); }
private void SavePages(VirtoData virtoData, ShopifyImportParams importParams, ShopifyImportNotification notification) { notification.Description = "Saving pages"; _notifier.Upsert(notification); var pageBasePath = "/Pages/" + importParams.StoreId + "/"; foreach (var page in virtoData.Pages) { var pageUrl = pageBasePath + page.Title + ".md"; using (var targetStream = _contentStorageProvider.OpenWrite(pageUrl)) { // convert string to stream byte[] byteArray = Encoding.UTF8.GetBytes(page.BodyHtml); using (var sourceStream = new MemoryStream(byteArray)) { sourceStream.CopyTo(targetStream); } } notification.Progresses[PagesKey].ProcessedCount++; _notifier.Upsert(notification); } }