Beispiel #1
0
        protected override void Process()
        {
            CurrentLanguage = PrimaryLanguage;
            var Soap = new AssortmentService();

#if CACHE
            if (!File.Exists(String.Format(ImagesCacheFile, Connector.ConnectorID, PrimaryLanguage.LanguageID)))
            {
                ImagesXml = XDocument.Parse(Soap.GetAssortmentImages(Connector.ConnectorID));
                ImagesXml.Save(String.Format(ImagesCacheFile, Connector.ConnectorID, PrimaryLanguage.LanguageID));
            }
            else
            {
                ImagesXml = XDocument.Load(String.Format(ImagesCacheFile, Connector.ConnectorID, PrimaryLanguage.LanguageID));
            }
#else
            ImagesXml = XDocument.Parse(Soap.GetAssortmentImages(Connector.ConnectorID));
#endif


            SBLicenseManager.TElSBLicenseManager m = new SBLicenseManager.TElSBLicenseManager();
            m.LicenseKey = "8F1317CD48AC4E68BABA5E339D8B365414D7ADA0289CA037E9074D29AD95FF3EC5D796BEFF0FBADB3BD82F48644C9EB810D9B5A305E0D2A1885C874D8BF974B9608CE918113FBE2AA5EEF8264C93B25ABEA98715DB4AD265F47CE02FC9952D69F2C3530B6ABAAA4C43B45E7EF6A8A0646DA038E34FBFB629C2BF0E83C6B348726E622EBD52CA05CF74C68F1279849CCD0C13EA673916BA42684015D658B8E7626F15BD826A4340EDB36CE55791A051FDBCF9FA1456C3B5008AD9990A0185C0EA3B19F9938CB7DA1FE82736ED4C7A566D4BFD53411E8380F4B020CB50E762520EFAE190836FD253B00DB18D4A485C7DC918AA4DCEC856331DD231CC8DC9C741C3";


            FtpInfo ftpInfo = SftpHelper.GetFtpInfo(Connector);


            Logger.DebugFormat("Exporting images for language '{0}'", CurrentLanguage.Language.Name);

            SortedDictionary <string, catalog_product_entity> currentProducts;
            SortedDictionary <string, eav_attribute>          attributeList;

            using (var helper = new MagentoMySqlHelper(Connector.Connection))
            {
                attributeList = helper.GetAttributeList(PRODUCT_ENTITY_TYPE_ID);

                currentProducts = helper.GetSkuList();
            }


            var products = (from i in ImagesXml.Root.Elements("Products").Elements("ProductMedia")
                            where i.Attribute("ManufacturerID") != null

                            orderby Convert.ToInt32(i.Attribute("ProductID").Value) descending
                            group i by i.Attribute("ManufacturerID").Value into grouped
                            select grouped).ToList();


            var directoryStructureRecords = (from i in ImagesXml.Root.Elements("Products").Elements("ProductMedia")
                                             // this catches the changes in webservice structure
                                             let urlAttribute = i.Attribute("Url")
                                                                let uri = (urlAttribute != null) ? new Uri(urlAttribute.Value) : new Uri(i.Value)
                                                                          select new
            {
                FirstLevel = uri.Segments.Last().Substring(0, 1),
                SecondLevel = uri.Segments.Last().Substring(1, 1)
            }).GroupBy(x => x.FirstLevel).ToDictionary(x => x.Key, y => y.Select(z => z.SecondLevel).Distinct());


            var dirClient = SftpHelper.GetSFTPClient(ftpInfo);


            foreach (var kv in directoryStructureRecords)
            {
                SftpHelper.EnsurePath(dirClient, ftpInfo.FtpPath + kv.Key);
                //if (!dirClient.Exists(kv.Key))
                //  dirClient.MakeDirectory(kv.Key);
                foreach (var sl in kv.Value)
                {
                    //if (!dirClient.Exists(kv.Key + "/" + sl))
                    //  dirClient.MakeDirectory(kv.Key + "/" + sl);
                    SftpHelper.EnsurePath(dirClient, ftpInfo.FtpPath + kv.Key + "/" + sl);
                }
            }

            dirClient.Close(true);

            int totalImages    = products.Select(x => x.Count()).Sum();
            int totalRecords   = products.Count;
            int totalProcessed = 0;

            if (totalRecords == 0)
            {
                Logger.DebugFormat("Finished exporting images for language '{0}'", CurrentLanguage.Language.Name);
                return;
            }
            ParallelOptions options = new ParallelOptions()
            {
                MaxDegreeOfParallelism = 8
            };

            Parallel.ForEach(Partitioner.Create(0, totalRecords), options, (range, loopState) =>
            {
                TElSimpleSFTPClient client = SftpHelper.GetSFTPClient(ftpInfo);
                try
                {
                    using (var helper = new AssortmentHelper(Connector.Connection, Version))
                    {
                        for (int index = range.Item1; index < range.Item2; index++)
                        {
                            var product = products[index];

                            var sku = product.Key.Trim();
                            catalog_product_entity entity;
                            if (!currentProducts.TryGetValue(sku, out entity))
                            {
                                continue;
                            }


                            List <string> currentPaths = new List <string>();

                            //helper.ResetGallery(entity, 0);

                            var sequencedImages = product.OrderBy(x => Convert.ToInt32(x.Attribute("Sequence").Value)).ToList();
                            var thumbnailImage  = sequencedImages.FirstOrDefault(c => c.Attribute("IsThumbnailImage") != null && Convert.ToBoolean(c.Attribute("IsThumbnailImage").Value));

                            int?thumbImageIdx = null;
                            if (thumbnailImage != null)
                            {
                                thumbImageIdx = sequencedImages.IndexOf(thumbnailImage);
                            }

                            for (int idx = 0; idx < sequencedImages.Count; idx++)
                            {
                                var image    = sequencedImages[idx];
                                int position = Convert.ToInt32(image.Attribute("Sequence").Value) + 1;

                                int storeid = 0;

                                // this catches the changes in webservice structure
                                var sourceUri = (image.Attribute("Url") != null) ? new Uri(image.Attribute("Url").Value) : new Uri(image.Value);
                                var label     = image.Attribute("Description") != null ? image.Attribute("Description").Value : string.Empty;

                                string fileName  = sourceUri.Segments.Last();
                                string linuxPath = String.Format(@"/{0}/{1}/{2}", fileName.Substring(0, 1),
                                                                 fileName.Substring(1, 1),
                                                                 fileName.Replace(" ", ""));

#if DEBUG
                                //sourceUri = new Uri(sourceUri.AbsoluteUri.Replace("localhost", "10.172.26.1"));
                                sourceUri = new Uri(sourceUri.AbsoluteUri.Replace("localhost", "172.16.250.94"));
#endif

                                string path = DownloadImage(sourceUri, client, ftpInfo.FtpPath, Logger);

                                if (path != null)
                                {
                                    helper.AddImageToGallery(entity.entity_id, storeid, path, position, label, Logger);

                                    if (idx == 0)
                                    {
                                        helper.SyncAttributeValue(attributeList["image"].attribute_id, PRODUCT_ENTITY_TYPE_ID, 0, entity.entity_id, path);
                                        helper.SyncAttributeValue(attributeList["small_image"].attribute_id, PRODUCT_ENTITY_TYPE_ID, 0, entity.entity_id, path);
                                        helper.SyncAttributeValue(attributeList["image_label"].attribute_id, PRODUCT_ENTITY_TYPE_ID, 0, entity.entity_id, label);
                                    }
                                    if (thumbImageIdx.HasValue && thumbImageIdx.Value == idx)
                                    {
                                        helper.SyncAttributeValue(attributeList["thumbnail"].attribute_id, PRODUCT_ENTITY_TYPE_ID, 0, entity.entity_id, path);
                                    }
                                    currentPaths.Add(path);
                                }
                            }

                            var currentImages = helper.GetGalleryItems(entity);

                            CleanupGallery(helper, entity, currentImages, currentPaths, client, ftpInfo.FtpPath);


                            Interlocked.Increment(ref totalProcessed);
                            //Logger.DebugFormat("Connections : {0}", ftp.ConnectionOpen);
                            //Logger.DebugFormat("Processed {0}", totalProcessed);
                            if (totalProcessed % 100 == 0)
                            {
                                Logger.DebugFormat(String.Format("Processed {0} of {1} products", totalProcessed, totalRecords));
                            }
                        }
                    }
                }
                catch (Exception e)
                {
                    Logger.Error("Image uploader error ", e);
                }
            });



            Logger.DebugFormat("Finished exporting images for language '{0}'", CurrentLanguage.Language.Name);
        }
Beispiel #2
0
        protected override void Process()
        {
            try
            {
                CurrentLanguage = PrimaryLanguage;

                var assortmentService = new AssortmentService();

#if CACHE
                if (File.Exists(String.Format(ImagesCacheFile, Connector.ConnectorID, PrimaryLanguage.LanguageID)))
                {
                    ImagesXml = XDocument.Load(String.Format(ImagesCacheFile, Connector.ConnectorID, PrimaryLanguage.LanguageID));
                }
                else
                {
                    ImagesXml = XDocument.Parse(assortmentService.GetAssortmentImages(Connector.ConnectorID));
                    ImagesXml.Save(String.Format(ImagesCacheFile, Connector.ConnectorID, PrimaryLanguage.LanguageID));
                }
#else
                ImagesXml = XDocument.Parse(assortmentService.GetAssortmentImages(Connector.ConnectorID));
#endif

                if (ImagesXml.Root.Element("Error") != null)
                {
                    Logger.Debug(ImagesXml);
                    throw new Exception("Web service failed call failed.");
                }

                //for debugging purposes
                var debugDirectoryPath = Path.Combine(_serializationPath, "Archive");
                var debugFilePath      = Path.Combine(debugDirectoryPath, string.Format(@"image_{0}_{1}.xml", Connector.ConnectorID, DateTime.Now.ToString("dd-MM-yyyy_hhmm")));
                try
                {
                    if (!Directory.Exists(debugDirectoryPath))
                    {
                        Directory.CreateDirectory(debugDirectoryPath);
                    }

                    ImagesXml.Save(debugFilePath);
                }
                catch (Exception e)
                {
                    Logger.Error("Archiving the xml failed in path " + debugFilePath, e);
                }

                var serializationPath = Path.Combine(_serializationPath, Connector.ConnectorID.ToString());
                if (!Directory.Exists(serializationPath))
                {
                    Directory.CreateDirectory(serializationPath);
                }

                ImageDiffComparisonHelper comparison = new ImageDiffComparisonHelper(ImagesXml, serializationPath);

                Logger.DebugFormat("Exporting images for language '{0}'", CurrentLanguage.Language.Name);

                var currentProducts = default(SortedDictionary <String, catalog_product_entity>);
                var attributeList   = default(SortedDictionary <String, eav_attribute>);

                using (var helper = new MagentoMySqlHelper(Connector.Connection))
                {
                    attributeList   = helper.GetAttributeList(PRODUCT_ENTITY_TYPE_ID);
                    currentProducts = helper.GetSkuList();
                }

                var products = comparison.ProductsToProcess
                               .Where(element => element.Attribute("ManufacturerID") != null)
                               .OrderByDescending(element => element.AttributeValue("ProductID", 0))
                               .GroupBy(element => element.AttributeValue("ManufacturerID"))
                               .ToArray();

                var directoryStructureRecords = (
                    from i in ImagesXml.Root.Elements("Products").Elements("ProductMedia")
                    let urlAttribute = i.Attribute("Url")
                                       let uri = (urlAttribute != null)
            ? new Uri(urlAttribute.Value)
            : new Uri(i.Value)
                                                 select new
                {
                    FirstLevel = uri.Segments.Last().Substring(0, 1),
                    SecondLevel = uri.Segments.Last().Substring(1, 1)
                })
                                                .GroupBy(x => x.FirstLevel)
                                                .ToDictionary(x => x.Key, y => y.Select(z => z.SecondLevel).Distinct());

                var mediaExchangeProvider = GetMediaExchangeProvider();

                int totalImages    = products.Select(element => element.Count()).Sum();
                int totalRecords   = products.Length;
                int totalProcessed = 0;

                if (totalRecords == 0)
                {
                    Logger.DebugFormat("Finished exporting images for language '{0}'", CurrentLanguage.Language.Name);
                    return;
                }

                var options = new ParallelOptions()
                {
                    MaxDegreeOfParallelism = 8
                };

                var basePath = Connector.ConnectorSettings.GetValueByKey <string>("FtpPath", string.Empty);
                if (basePath.EndsWith("/"))
                {
                    basePath = basePath.TrimEnd('/');
                }


                Parallel.For(0, totalRecords, options, index =>
                {
                    using (var helper = new AssortmentHelper(Connector.Connection, Version))
                    {
                        var product = products[index];
                        var sku     = product.Key.Trim();
                        var entity  = default(catalog_product_entity);

                        try
                        {
                            if (currentProducts.TryGetValue(sku, out entity))
                            {
                                var currentPathList = new List <String>();

                                var sequencedImages = product.OrderBy(x => x.AttributeValue("Sequence", 0)).ToList();
                                var thumbnailImage  = sequencedImages.FirstOrDefault(c => c.AttributeValue("IsThumbnailImage", false));

                                int?thumbImageIndex = null;

                                if (thumbnailImage != null)
                                {
                                    thumbImageIndex = sequencedImages.IndexOf(thumbnailImage);
                                }

                                for (var imageIndex = 0; imageIndex < sequencedImages.Count; imageIndex++)
                                {
                                    var storeID   = 0;
                                    var image     = sequencedImages[imageIndex];
                                    var position  = image.AttributeValue("Sequence", 0) + 1;
                                    var sourceUri = new Uri(image.AttributeValue("Url", image.Value));
                                    var label     = image.AttributeValue("Description", String.Empty);

                                    var fileName = HttpUtility.UrlDecode(sourceUri.Segments.Last());

                                    fileName = fileName.Replace(" ", "");

                                    var remotePath = String.Format(@"/{0}/{1}/{2}"
                                                                   , fileName.Substring(0, 1)
                                                                   , fileName.Substring(1, 1)
                                                                   , fileName.Replace(" ", String.Empty));


                                    var path = basePath + remotePath;

                                    //#if DEBUG
                                    //                  //sourceUri = new Uri(sourceUri.AbsoluteUri.Replace("localhost", "10.172.26.1"));
                                    //                  sourceUri = new Uri(sourceUri.AbsoluteUri.Replace("localhost", "54.72.108.185"));
                                    //#endif

                                    using (WebClient dlClient = new WebClient())
                                    {
                                        using (var stream = new MemoryStream(dlClient.DownloadData(sourceUri)))
                                        {
                                            mediaExchangeProvider.Upload(stream, path);
                                        }
                                    }

                                    if (remotePath != null)
                                    {
                                        helper.AddImageToGallery(entity.entity_id, storeID, remotePath, position, label, Logger);

                                        if (imageIndex == 0)
                                        {
                                            helper.SyncAttributeValue(attributeList["image"].attribute_id, PRODUCT_ENTITY_TYPE_ID, 0, entity.entity_id, remotePath);
                                            helper.SyncAttributeValue(attributeList["small_image"].attribute_id, PRODUCT_ENTITY_TYPE_ID, 0, entity.entity_id, remotePath);
                                            helper.SyncAttributeValue(attributeList["image_label"].attribute_id, PRODUCT_ENTITY_TYPE_ID, 0, entity.entity_id, label);
                                        }

                                        if (thumbImageIndex.HasValue && thumbImageIndex.Value == imageIndex)
                                        {
                                            helper.SyncAttributeValue(attributeList["thumbnail"].attribute_id, PRODUCT_ENTITY_TYPE_ID, 0, entity.entity_id, remotePath);
                                        }

                                        currentPathList.Add(remotePath);
                                    }
                                }

                                var mediaGalleryItems = helper
                                                        .GetGalleryItems(entity)
                                                        .Where(mediaGalleryItem => !currentPathList.Contains(mediaGalleryItem.value))
                                                        .ToArray();

                                foreach (var file in mediaGalleryItems.Select(mediaGalleryItem => basePath + mediaGalleryItem.value))
                                {
                                    try
                                    {
                                        mediaExchangeProvider.Delete(file);
                                    }
                                    catch
                                    {
                                    }
                                }

                                helper.DeleteGalleryItems(mediaGalleryItems);

                                Interlocked.Increment(ref totalProcessed);

                                if (totalProcessed % 100 == 0)
                                {
                                    Logger.DebugFormat(String.Format("Processed {0} of {1} products", totalProcessed, totalRecords));
                                }
                                helper.SetProductLastModificationTime(entity.entity_id);
                            }
                            else
                            {
                                comparison.MarkProductAsNotProcessed(product.ToList());
                            }
                        }
                        catch (Exception e)
                        {
                            Logger.Error(string.Format("Image uploader error for sku {0}", sku), e);

                            comparison.MarkProductAsNotProcessed(product.ToList());
                        }
                    }
                });

                Logger.DebugFormat("Finished exporting images for language '{0}'", CurrentLanguage.Language.Name);
                comparison.ArchiveImages();

                mediaExchangeProvider.Dispose();
            }
            catch (Exception e)
            {
                Logger.Error("Image exporter failed", e);
                throw e;
            }
        }