コード例 #1
0
ファイル: frmMain.cs プロジェクト: ruslanruslanruslan/Parser
        private void LoadSection(string[] linkSection)
        {
            /*if (!string.IsNullOrEmpty(LinkAdtextBox.Text) && !string.IsNullOrEmpty(userNametextBox.Text)
             && !string.IsNullOrEmpty(PasswordtextBox.Text) && !string.IsNullOrEmpty(pathToProxytextBox.Text)
             && !string.IsNullOrEmpty(pathToImgtextBox.Text))*/
              {
            try
            {
              label6.SetPropertyThreadSafe(() => label6.Text, "Start");

              while (AvitoState.Pausing || AvitoState.Paused)
              {
            if (AvitoState.Pausing)
            {
              AddLog(string.Empty, LogMessageColor.Information());
              AddLog("Paused", LogMessageColor.Information());
              AddLog(string.Empty, LogMessageColor.Information());
            }
            AvitoState.SetPaused();
            if (EbayState.Pausing || EbayState.Paused)
            {
              btnParsingAvitoEbayStart.SetPropertyThreadSafe(() => btnParsingAvitoEbayStart.Enabled, true);
              btnParsingAvitoEbayPause.SetPropertyThreadSafe(() => btnParsingAvitoEbayPause.Enabled, false);
              btnParsingAvitoEbayStop.SetPropertyThreadSafe(() => btnParsingAvitoEbayStop.Enabled, true);
            }
            else
            {
              btnParsingAvitoStart.SetPropertyThreadSafe(() => btnParsingAvitoStart.Enabled, true);
              btnParsingAvitoPause.SetPropertyThreadSafe(() => btnParsingAvitoPause.Enabled, false);
              btnParsingAvitoStop.SetPropertyThreadSafe(() => btnParsingAvitoStop.Enabled, true);
            }
            Thread.Sleep(1000);
              }

              if (AvitoState.Stopping || AvitoState.Stopped)
              {
            label6.SetPropertyThreadSafe(() => label6.Text, "Finish");
            return;
              }

              //Ссылка на обьявление
              //  URLLink = LinkAdtextBox.Text;
              //Создаем класс и вводим параметры
              var Parser = new RuslanParser(Properties.Default.User, Properties.Default.Password, Properties.Default.PathToProxy, mySqlDB);
              var imageCount = new ImageParsedCountHelper();
              var Parser2 = new RuslanParser2(Properties.Default.User, Properties.Default.Password, Properties.Default.PathToProxy, mySqlDB, Properties.Default.FtpUsername, Properties.Default.FtpPassword, imageCount);

              Parser.PathImages = Properties.Default.PathToImg;

              var linksAds = Parser.LoadLinks(linkSection[0]);
              if (linksAds == null)
              {
            AddLog("Parser: Couldn't retrieve ad links from given section " + linkSection[1], LogMessageColor.Error());
            return;
              }
              AddLog("Parser: " + linkSection[1], LogMessageColor.Information());
              AddLog("Parser: Count new ad: " + linksAds.Count().ToString(), LogMessageColor.Information());
              var i = 0;
              var countPre = 0;
              var countIns = 0;

              while (AvitoState.Pausing || AvitoState.Paused)
              {
            if (AvitoState.Pausing)
            {
              AddLog(string.Empty, LogMessageColor.Information());
              AddLog("Paused", LogMessageColor.Information());
              AddLog(string.Empty, LogMessageColor.Information());
            }
            AvitoState.SetPaused();
            if (EbayState.Pausing || EbayState.Paused)
            {
              btnParsingAvitoEbayStart.SetPropertyThreadSafe(() => btnParsingAvitoEbayStart.Enabled, true);
              btnParsingAvitoEbayPause.SetPropertyThreadSafe(() => btnParsingAvitoEbayPause.Enabled, false);
              btnParsingAvitoEbayStop.SetPropertyThreadSafe(() => btnParsingAvitoEbayStop.Enabled, true);
            }
            else
            {
              btnParsingAvitoStart.SetPropertyThreadSafe(() => btnParsingAvitoStart.Enabled, true);
              btnParsingAvitoPause.SetPropertyThreadSafe(() => btnParsingAvitoPause.Enabled, false);
              btnParsingAvitoStop.SetPropertyThreadSafe(() => btnParsingAvitoStop.Enabled, true);
            }
            Thread.Sleep(1000);
              }

              if (AvitoState.Stopping || AvitoState.Stopped)
              {
            label6.SetPropertyThreadSafe(() => label6.Text, "Finish");
            return;
              }

              foreach (var item in linksAds)
              {
            while (AvitoState.Pausing || AvitoState.Paused)
            {
              if (AvitoState.Pausing)
              {
                AddLog(string.Empty, LogMessageColor.Information());
                AddLog("Paused", LogMessageColor.Information());
                AddLog(string.Empty, LogMessageColor.Information());
              }
              AvitoState.SetPaused();
              if (EbayState.Pausing || EbayState.Paused)
              {
                btnParsingAvitoEbayStart.SetPropertyThreadSafe(() => btnParsingAvitoEbayStart.Enabled, true);
                btnParsingAvitoEbayPause.SetPropertyThreadSafe(() => btnParsingAvitoEbayPause.Enabled, false);
                btnParsingAvitoEbayStop.SetPropertyThreadSafe(() => btnParsingAvitoEbayStop.Enabled, true);
              }
              else
              {
                btnParsingAvitoStart.SetPropertyThreadSafe(() => btnParsingAvitoStart.Enabled, true);
                btnParsingAvitoPause.SetPropertyThreadSafe(() => btnParsingAvitoPause.Enabled, false);
                btnParsingAvitoStop.SetPropertyThreadSafe(() => btnParsingAvitoStop.Enabled, true);
              }
              Thread.Sleep(1000);
            }

            if (AvitoState.Stopping || AvitoState.Stopped)
              break;

            try
            {
              imageCount.ErrorList.Clear();
              imageCount.Resources.Clear();
              imageCount.CountParsed = 0;
              imageCount.CountDownloaded = 0;
              imageCount.ResourceId = string.Empty;
              AddLog("Parser: start parse link: " + item, LogMessageColor.Information());
              i++;
              if (i == 25)
                i = -1;
              URLLink = item;
              mySqlDB.PrepareAvitoEnvironment();
              Dictionary<PartsPage, IEnumerable<string>> result = null;
              try
              {
                result = Parser.Run(item);
              }
              catch (Exception ex)
              {
                AddLog("Parser: " + ex.Message, LogMessageColor.Error());
                continue;
              }
              var sb = new StringBuilder();
              sb.AppendLine();
              foreach (var element in result)
              {
                if (element.Key != PartsPage.Body)
                {
                  sb.Append(string.Format("{0,-15} - ", element.Key));
                  if (element.Value != null)
                    foreach (var t in element.Value)
                    {
                      if (element.Value.Count() > 1)
                        sb.Append(t + " |");
                      else
                        sb.Append(t);
                    }
                  sb.Append(Environment.NewLine);
                }
              }
              AddLog("Parser: " + sb.ToString().TrimEnd(Environment.NewLine.ToCharArray()), LogMessageColor.Information());
              IncParsed();
              countPre++;

              AddLog("Parser: sleep after parse on. " + Properties.Default.SleepAfterParseSec + " sec", LogMessageColor.Information());
              Thread.Sleep(Properties.Default.SleepAfterParseSec * 1000);
              AddLog("Parser: sleep after parse off", LogMessageColor.Information());

              AddLog("Parser: preparing ad to insert to db", LogMessageColor.Information());
              var idResourceList = mySqlDB.ResourceListIDAvito();
              mySqlDB.InsertFctAvitoGrabber(result, idResourceList, item, linkSection[1]);
              AddLog("Parser: ad inserted", LogMessageColor.Information());
              incInserted();
              imageCount.ResourceId = idResourceList;
              if (Properties.Default.PublishParsedData)
              {
                AddLog("Parser: Begin loading images", LogMessageColor.Information());
                Parser2.PathImages2 = Properties.Default.PathToImg;

                var result2 = Parser2.Run(item);
                AddLog("Parser: " + imageCount.CountParsed + " images parsed", LogMessageColor.Information());
                AddLog("Parser: " + imageCount.CountDownloaded + " images downloaded", LogMessageColor.Information());

                foreach (var error in imageCount.ErrorList)
                  AddLog(error.Key, error.Value == true ? LogMessageColor.Error() : LogMessageColor.Success());

                foreach (var im in imageCount.Resources)
                  mySqlDB.InsertassGrabberAvitoResourceList(idResourceList, im);

                mySqlDB.UpdateFctAvitoGrabberPhotoCount(idResourceList, imageCount.Resources.Count);

                AddLog("Parser: Loading images end", LogMessageColor.Information());
                AddLog("Parser: Begin publishing ad", LogMessageColor.Information());
                mySqlDB.ExecuteProcAvito(idResourceList);
                AddLog("Parser: End publishing ad", LogMessageColor.Information());

                AddLog("Parser: sleep after publication on. " + Properties.Default.SleepSecAfterPublicationSec + " sec", LogMessageColor.Information());
                Thread.Sleep(Properties.Default.SleepSecAfterPublicationSec * 1000);
                AddLog("Parser: sleep after publication off", LogMessageColor.Information());
              }
              else
              {
                AddLog("Parser: Saving data to SMSSpamer", LogMessageColor.Information());
                mySqlDB.InsertSMSSpamerData(idResourceList);
                AddLog("Parser: Data saved successfully", LogMessageColor.Information());
              }
              countIns++;
            }
            catch (Exception ex)
            {
              AddLog("Parser: " + ex.Message, LogMessageColor.Error());
            }
            AddLog(string.Empty, LogMessageColor.Information());
              }
              AddLogStatistic(linkSection[1], mySqlDB.CountAd, countIns);
            }
            catch (Exception ex)
            {
              AddLog("Parser: " + ex.Message, LogMessageColor.Error());
            }
            label6.SetPropertyThreadSafe(() => label6.Text, "Finish");
              }
        }
コード例 #2
0
ファイル: frmMain.cs プロジェクト: ruslanruslanruslan/Parser
        private void OneLinkAvito()
        {
            try
              {
            label6.SetPropertyThreadSafe(() => label6.Text, "Start");
            //Ссылка на обьявление
            mySqlDB.PrepareAvitoEnvironment();
            //Создаем класс и вводим параметры
            var Parser = new RuslanParser(Properties.Default.User, Properties.Default.Password, Properties.Default.PathToProxy, mySqlDB);
            Parser.PathImages = Properties.Default.PathToImg;
            //вот тут мы вызывем запрос на Id к базе
            //Parser.LoadGuid = (() => MySqlDB.SeletGuid());
            //Parser.LoadGuid = (() => "1532");
            //тут мы присваем результат переменной result
            Dictionary<PartsPage, IEnumerable<string>> result = null;
            try
            {
              result = Parser.Run(URLLink);
            }
            catch (Exception ex)
            {
              AddLog("Parser: " + ex.Message, LogMessageColor.Error());
            }
            // тут мы передаем в метод вставки данные
            //result, index ID, ссылку на обьявления
            //id я не вставлял так как непонятно было и неподходило под структуру бд
            if (result != null)
            {
              var idResourceList = mySqlDB.ResourceListIDAvito();
              mySqlDB.InsertFctAvitoGrabber(result, idResourceList, URLLink, cbCategories.Text);
              var imageCount = new ImageParsedCountHelper();
              imageCount.ResourceId = idResourceList;
              var Parser2 = new RuslanParser2(Properties.Default.User, Properties.Default.Password, Properties.Default.PathToProxy, mySqlDB, Properties.Default.FtpUsername, Properties.Default.FtpPassword, imageCount);
              Parser2.PathImages2 = Properties.Default.PathToImg;
              var result2 = Parser2.Run(URLLink);
              AddLog("Parser: " + imageCount.CountParsed + " images parsed", LogMessageColor.Information());
              AddLog("Parser: " + imageCount.CountDownloaded + " images downloaded", LogMessageColor.Information());
              foreach (var error in imageCount.ErrorList)
              {
            AddLog(error.Key, error.Value == true ? LogMessageColor.Error() : LogMessageColor.Success());
              }

              foreach (var im in imageCount.Resources)
            mySqlDB.InsertassGrabberAvitoResourceList(idResourceList, im);

              mySqlDB.UpdateFctAvitoGrabberPhotoCount(idResourceList, imageCount.Resources.Count);

              AddLog("Parser: Loading images end", LogMessageColor.Information());
              AddLog("Parser: Begin publishing ad", LogMessageColor.Information());
              mySqlDB.ExecuteProcAvito(idResourceList);
              AddLog("Parser: End publishing ad", LogMessageColor.Information());
            }
              }
              catch (Exception ex)
              {
            AddLog("Parser: " + ex.Message, LogMessageColor.Error());
              }
              label6.SetPropertyThreadSafe(() => label6.Text, "Finish");
        }