Esempio n. 1
0
        static void Main(string[] args)
        {
            CFlipkartScrapper objscrapper = new CFlipkartScrapper();
            ItemDescription   objItem     = new MicrowaveItemDescription();

            objscrapper.SetHTMLDocument("http://www.flipkart.com/samsung-40h5100-40-inches-57900-tv/p/itmdu7ycguexxhzm?pid=TVSDU5QTVF9BX57Z");
            objscrapper.ExtractData(ref objItem);



            string path     = "DealSheelImages//" + objItem.getLabel();
            string fileName = objItem.id + ImageController.GetFileTypeFromImage(objItem.image);

            ImageController.GetImageFromUrlAndSave(objItem.image, fileName);

            ImageController.SaveimageToRemoteLocation(path, fileName, "localhost", "8080");
        }
Esempio n. 2
0
            public void ScrapModelNumberAndWriteToCSV()
            {
                //scrap the model number
                CFlipkartScrapper objscrapper = new CFlipkartScrapper();



                foreach (ProductStructure objpr in m_lProduct)
                {
                    //Get the product id and search if its already there in the csv
                    if (m_dictProduct.ContainsKey(objpr.productId))
                    {
                        continue;
                    }

                    try
                    {
                        ItemDescription objItem = new ItemDescription();

                        objscrapper.SetHTMLDocument(objpr.productUrl);
                        objscrapper.ExtractData(ref objItem);

                        if (objItem.Model_ID != null)
                        {
                            objpr.Model_Name = objItem.Model_ID;
                        }
                        else
                        {
                            objpr.Model_Name = objItem.Model_Name;
                        }

                        //Write to csv file
                        string line = objpr.Model_Name + Constants.CSV_DELIMITER + objpr.productBrand + Constants.CSV_DELIMITER + objpr.productUrl + Constants.CSV_DELIMITER + objpr.productId;
                        Logger.WriteToCSVFile(line, m_fileName, System.IO.Path.GetDirectoryName(m_fileName));
                    }
                    catch (Exception ex)
                    {
                    }
                }

                m_threadCount--;
            }
Esempio n. 3
0
        public static ExcelStructure SaveToDB(ExcelStructure objExcelStruct, Store objStore, SubCategory objSubCategory, Brand objBrand, string server, string port, GridViewRowInfo row = null)
        {
            try
            {
                if (objStore == null)
                {
                    throw new Exception("Please select a store .....");
                }
                if (objSubCategory == null)
                {
                    throw new Exception("Subcateogry do not match or is empty");
                }
                if (objBrand == null)
                {
                    throw new Exception("Please add brand for this subcategory ir brand is null or not matching");
                }


                ItemDescription objItemDescr = DBGetInterface.GetItemDescription(objBrand, objExcelStruct.ModelNumber);

                if (objItemDescr == null)
                {
                    //Call flipkart scrapper

                    CFlipkartScrapper objscrapper = new CFlipkartScrapper();
                    ItemDescription   objItemdesc = ItemFactory.GetItem(objSubCategory);

                    if (String.IsNullOrEmpty(objExcelStruct.FlipKart))
                    {
                        //check if the item link is already scrapped, find in csv
                        string strFilePath = m_ResourcesFolder + "//" + m_CategoryMappingTxt;
                        if (System.IO.File.Exists(strFilePath))
                        {
                            //File exists
                            //read the file and check for that file in LinkFodler
                            try
                            {
                                using (System.IO.StreamReader sw = new System.IO.StreamReader(strFilePath))
                                {
                                    string line = String.Empty;
                                    while ((line = sw.ReadLine()) != null)
                                    {
                                        bool     bflag = false;
                                        string[] cat   = line.Split(' ');
                                        if (cat[0].Equals(objSubCategory.SubCategoryID.ToString(), StringComparison.InvariantCultureIgnoreCase))
                                        {
                                            //search model number in that subcategory file
                                            string file = Constants.LINK_EXTRACTED_FOLDER + "//" + cat[1] + ".csv";

                                            //search in this csv file
                                            List <List <string> > lmap = Utilities.GetCSVSheet(file);
                                            foreach (List <string> product in lmap)
                                            {
                                                if (product.ElementAt(0).Equals(objExcelStruct.ModelNumber, StringComparison.InvariantCultureIgnoreCase))
                                                {
                                                    objExcelStruct.FlipKart = product.ElementAt(2);
                                                    bflag = true;
                                                    break;
                                                }
                                            }
                                        }

                                        if (bflag)
                                        {
                                            break;
                                        }
                                    }
                                }
                            }
                            catch (Exception ex)
                            {
                            }
                        }
                    }

                    objscrapper.SetHTMLDocument(objExcelStruct.FlipKart);
                    objscrapper.ExtractData(ref objItemdesc);

                    string image = objItemdesc.image;
                    objItemdesc.image = null;

                    //Now do the following
                    //Create item, create item description.
                    Item objItem = new Item();
                    objItem.Price            = Convert.ToUInt32(objExcelStruct.Price);
                    objItem.OfferDescription = objExcelStruct.Offer;

                    ItemWrapper objWrap = new ItemWrapper();
                    objWrap.objBrand           = objBrand;
                    objWrap.objStore           = objStore;
                    objWrap.objItemDescription = objItemdesc;
                    objWrap.objItem            = objItem;
                    DBAddinterface.CreateItemNode(objWrap);

                    //Copy image
                    //Create a seperate thread for it as image copying is a tedious process
                    //Copy image to local machine and then copy it to tomcat machine
                    //build path
                    string imageDir = Utilities.GetImageDir(objItemdesc.getLabel());
                    Utilities.CreateFolder(imageDir);
                    string imagePath = Utilities.GetImagePath(imageDir, objItemdesc.id);
                    ImageController.GetImageFromUrlAndSave(image, imagePath);


                    //Create a seperate thread for this to do
                    Thread SaveImageThread = new Thread(() => SaveImageToRemoteMachineThread(imageDir, Utilities.AppendImageType(objItemdesc.id), server, port));
                    SaveImageThread.Start();


                    objExcelStruct.Completed = Constants.COMPLETED_DONE;
                    objExcelStruct.Comment   = String.Empty;
                }
                else
                {
                    objExcelStruct.Comment = Constants.ALREADY_IN_DB;
                    //Just update the price and offer of the item
                    Item objItem = DBGetInterface.GetItem(objStore, objItemDescr);
                    if (objItem == null)
                    {
                        //Add this node
                        objItem                  = new Item();
                        objItem.Price            = Convert.ToUInt32(objExcelStruct.Price);
                        objItem.OfferDescription = objExcelStruct.Offer;

                        ItemWrapper objWrap = new ItemWrapper();
                        objWrap.objBrand           = objBrand;
                        objWrap.objStore           = objStore;
                        objWrap.objItemDescription = objItemDescr;
                        objWrap.objItem            = objItem;
                        DBAddinterface.CreateItemNode(objWrap);

                        objExcelStruct.Completed = Constants.COMPLETED_ADDED;
                    }
                    else
                    {
                        //Check for any updates
                        bool bUpdate = false;
                        if (objItem.Price != Convert.ToUInt32(objExcelStruct.Price))
                        {
                            objItem.Price = Convert.ToUInt32(objExcelStruct.Price);
                            bUpdate       = true;
                        }

                        string offer = String.Empty;
                        if (objExcelStruct.Offer != null)
                        {
                            offer = objExcelStruct.Offer;
                        }

                        if (!objItem.OfferDescription.Equals(offer, StringComparison.InvariantCultureIgnoreCase))
                        {
                            objItem.OfferDescription = objExcelStruct.Offer;
                            bUpdate = true;
                        }

                        if (bUpdate)
                        {
                            objItem.lastUpdated = Utilities.GetDateTimeInUnixTimeStamp();
                            DBUpdateInterface.UpdateNode <Item>(objItem);
                            objExcelStruct.Completed = Constants.COMPLETED_UPDATED;
                        }
                        else
                        {
                            objExcelStruct.Completed = Constants.COMPLETED_NOUPDATES;
                        }
                    }
                }
            }
            catch (Exception ex)
            {
                objExcelStruct.Completed = Constants.COMPLETED_ERROR;
                objExcelStruct.Comment   = ex.Message;
            }

            if (row != null)
            {
                row.Cells["Completed"].Value = objExcelStruct.Completed;
                row.Cells["Comment"].Value   = objExcelStruct.Comment;
            }

            return(objExcelStruct);
        }