public ImageParsedCountHelper LoadImages(IEnumerable<string> LinksImages) { var helper = new ImageParsedCountHelper(); if (LinksImages != null) { helper.CountParsed = LinksImages.Count(); foreach (var item in LinksImages) { var guid = mySqlDB.ResourceID(); try { var dirName = string.Empty; if (guid.Length > 3) dirName = guid.Substring(0, 3); else dirName = guid; if (imageLoader.LoadImage(item, web, guid, helper.ResourceId, dirName)) { mySqlDB.InsertItemResource(guid, frmMain.URLLink, dirName); helper.Resources.Add(guid); helper.CountDownloaded++; helper.ErrorList.Add("LoadImage success: " + item, false); } } catch (Exception ex) { helper.ErrorList.Add("LoadImage error: " + item + ": " + ex.Message, true); } } } return helper; }
public RuslanParser2(string user, string pass, string pathToProxy, MySqlDB _mySqlDB, string _ftpUsername, string _ftpPassword, ImageParsedCountHelper imageParsed) { mySqlDB = _mySqlDB; ftpUsername = _ftpUsername; ftpPassword = _ftpPassword; imageParsedCountHelper = imageParsed; }
public AvitoLoadImageDeferentSize(IHttpWeb httpweb, string pathFolder, MySqlDB _mySqlDB, string _ftpUsername, string _ftpPassword, ImageParsedCountHelper imageParsed) : base(httpweb, pathFolder, imageParsed) { mySqlDB = _mySqlDB; ftpUsername = _ftpUsername; ftpPassword = _ftpPassword; imageLoader = new ImageLoader(PathToFolder, ftpUsername, ftpPassword); }
public AvitoLoadImages(IHttpWeb httpweb, string pathFolder, ImageParsedCountHelper imageCount = null) { imageParsedCountHelper = imageCount; WebCl = httpweb; PathToFolder = pathFolder.TrimEnd('/', '\\'); if ((pathFolder.ToLower()).StartsWith("ftp://") == false) { var isEsist = Directory.Exists(pathFolder); { if (!isEsist) throw new FileNotFoundException("LoadImage error: Path no exists '" + pathFolder + "'"); } } }
private void LoadSection(string[] linkSection) { /*if (!string.IsNullOrEmpty(LinkAdtextBox.Text) && !string.IsNullOrEmpty(userNametextBox.Text) && !string.IsNullOrEmpty(PasswordtextBox.Text) && !string.IsNullOrEmpty(pathToProxytextBox.Text) && !string.IsNullOrEmpty(pathToImgtextBox.Text))*/ { try { label6.SetPropertyThreadSafe(() => label6.Text, "Start"); while (AvitoState.Pausing || AvitoState.Paused) { if (AvitoState.Pausing) { AddLog(string.Empty, LogMessageColor.Information()); AddLog("Paused", LogMessageColor.Information()); AddLog(string.Empty, LogMessageColor.Information()); } AvitoState.SetPaused(); if (EbayState.Pausing || EbayState.Paused) { btnParsingAvitoEbayStart.SetPropertyThreadSafe(() => btnParsingAvitoEbayStart.Enabled, true); btnParsingAvitoEbayPause.SetPropertyThreadSafe(() => btnParsingAvitoEbayPause.Enabled, false); btnParsingAvitoEbayStop.SetPropertyThreadSafe(() => btnParsingAvitoEbayStop.Enabled, true); } else { btnParsingAvitoStart.SetPropertyThreadSafe(() => btnParsingAvitoStart.Enabled, true); btnParsingAvitoPause.SetPropertyThreadSafe(() => btnParsingAvitoPause.Enabled, false); btnParsingAvitoStop.SetPropertyThreadSafe(() => btnParsingAvitoStop.Enabled, true); } Thread.Sleep(1000); } if (AvitoState.Stopping || AvitoState.Stopped) { label6.SetPropertyThreadSafe(() => label6.Text, "Finish"); return; } //Ссылка на обьявление // URLLink = LinkAdtextBox.Text; //Создаем класс и вводим параметры var Parser = new RuslanParser(Properties.Default.User, Properties.Default.Password, Properties.Default.PathToProxy, mySqlDB); var imageCount = new ImageParsedCountHelper(); var Parser2 = new RuslanParser2(Properties.Default.User, Properties.Default.Password, Properties.Default.PathToProxy, mySqlDB, Properties.Default.FtpUsername, Properties.Default.FtpPassword, imageCount); Parser.PathImages = Properties.Default.PathToImg; var linksAds = Parser.LoadLinks(linkSection[0]); if (linksAds == null) { AddLog("Parser: Couldn't retrieve ad links from given section " + linkSection[1], LogMessageColor.Error()); return; } AddLog("Parser: " + linkSection[1], LogMessageColor.Information()); AddLog("Parser: Count new ad: " + linksAds.Count().ToString(), LogMessageColor.Information()); var i = 0; var countPre = 0; var countIns = 0; while (AvitoState.Pausing || AvitoState.Paused) { if (AvitoState.Pausing) { AddLog(string.Empty, LogMessageColor.Information()); AddLog("Paused", LogMessageColor.Information()); AddLog(string.Empty, LogMessageColor.Information()); } AvitoState.SetPaused(); if (EbayState.Pausing || EbayState.Paused) { btnParsingAvitoEbayStart.SetPropertyThreadSafe(() => btnParsingAvitoEbayStart.Enabled, true); btnParsingAvitoEbayPause.SetPropertyThreadSafe(() => btnParsingAvitoEbayPause.Enabled, false); btnParsingAvitoEbayStop.SetPropertyThreadSafe(() => btnParsingAvitoEbayStop.Enabled, true); } else { btnParsingAvitoStart.SetPropertyThreadSafe(() => btnParsingAvitoStart.Enabled, true); btnParsingAvitoPause.SetPropertyThreadSafe(() => btnParsingAvitoPause.Enabled, false); btnParsingAvitoStop.SetPropertyThreadSafe(() => btnParsingAvitoStop.Enabled, true); } Thread.Sleep(1000); } if (AvitoState.Stopping || AvitoState.Stopped) { label6.SetPropertyThreadSafe(() => label6.Text, "Finish"); return; } foreach (var item in linksAds) { while (AvitoState.Pausing || AvitoState.Paused) { if (AvitoState.Pausing) { AddLog(string.Empty, LogMessageColor.Information()); AddLog("Paused", LogMessageColor.Information()); AddLog(string.Empty, LogMessageColor.Information()); } AvitoState.SetPaused(); if (EbayState.Pausing || EbayState.Paused) { btnParsingAvitoEbayStart.SetPropertyThreadSafe(() => btnParsingAvitoEbayStart.Enabled, true); btnParsingAvitoEbayPause.SetPropertyThreadSafe(() => btnParsingAvitoEbayPause.Enabled, false); btnParsingAvitoEbayStop.SetPropertyThreadSafe(() => btnParsingAvitoEbayStop.Enabled, true); } else { btnParsingAvitoStart.SetPropertyThreadSafe(() => btnParsingAvitoStart.Enabled, true); btnParsingAvitoPause.SetPropertyThreadSafe(() => btnParsingAvitoPause.Enabled, false); btnParsingAvitoStop.SetPropertyThreadSafe(() => btnParsingAvitoStop.Enabled, true); } Thread.Sleep(1000); } if (AvitoState.Stopping || AvitoState.Stopped) break; try { imageCount.ErrorList.Clear(); imageCount.Resources.Clear(); imageCount.CountParsed = 0; imageCount.CountDownloaded = 0; imageCount.ResourceId = string.Empty; AddLog("Parser: start parse link: " + item, LogMessageColor.Information()); i++; if (i == 25) i = -1; URLLink = item; mySqlDB.PrepareAvitoEnvironment(); Dictionary<PartsPage, IEnumerable<string>> result = null; try { result = Parser.Run(item); } catch (Exception ex) { AddLog("Parser: " + ex.Message, LogMessageColor.Error()); continue; } var sb = new StringBuilder(); sb.AppendLine(); foreach (var element in result) { if (element.Key != PartsPage.Body) { sb.Append(string.Format("{0,-15} - ", element.Key)); if (element.Value != null) foreach (var t in element.Value) { if (element.Value.Count() > 1) sb.Append(t + " |"); else sb.Append(t); } sb.Append(Environment.NewLine); } } AddLog("Parser: " + sb.ToString().TrimEnd(Environment.NewLine.ToCharArray()), LogMessageColor.Information()); IncParsed(); countPre++; AddLog("Parser: sleep after parse on. " + Properties.Default.SleepAfterParseSec + " sec", LogMessageColor.Information()); Thread.Sleep(Properties.Default.SleepAfterParseSec * 1000); AddLog("Parser: sleep after parse off", LogMessageColor.Information()); AddLog("Parser: preparing ad to insert to db", LogMessageColor.Information()); var idResourceList = mySqlDB.ResourceListIDAvito(); mySqlDB.InsertFctAvitoGrabber(result, idResourceList, item, linkSection[1]); AddLog("Parser: ad inserted", LogMessageColor.Information()); incInserted(); imageCount.ResourceId = idResourceList; if (Properties.Default.PublishParsedData) { AddLog("Parser: Begin loading images", LogMessageColor.Information()); Parser2.PathImages2 = Properties.Default.PathToImg; var result2 = Parser2.Run(item); AddLog("Parser: " + imageCount.CountParsed + " images parsed", LogMessageColor.Information()); AddLog("Parser: " + imageCount.CountDownloaded + " images downloaded", LogMessageColor.Information()); foreach (var error in imageCount.ErrorList) AddLog(error.Key, error.Value == true ? LogMessageColor.Error() : LogMessageColor.Success()); foreach (var im in imageCount.Resources) mySqlDB.InsertassGrabberAvitoResourceList(idResourceList, im); mySqlDB.UpdateFctAvitoGrabberPhotoCount(idResourceList, imageCount.Resources.Count); AddLog("Parser: Loading images end", LogMessageColor.Information()); AddLog("Parser: Begin publishing ad", LogMessageColor.Information()); mySqlDB.ExecuteProcAvito(idResourceList); AddLog("Parser: End publishing ad", LogMessageColor.Information()); AddLog("Parser: sleep after publication on. " + Properties.Default.SleepSecAfterPublicationSec + " sec", LogMessageColor.Information()); Thread.Sleep(Properties.Default.SleepSecAfterPublicationSec * 1000); AddLog("Parser: sleep after publication off", LogMessageColor.Information()); } else { AddLog("Parser: Saving data to SMSSpamer", LogMessageColor.Information()); mySqlDB.InsertSMSSpamerData(idResourceList); AddLog("Parser: Data saved successfully", LogMessageColor.Information()); } countIns++; } catch (Exception ex) { AddLog("Parser: " + ex.Message, LogMessageColor.Error()); } AddLog(string.Empty, LogMessageColor.Information()); } AddLogStatistic(linkSection[1], mySqlDB.CountAd, countIns); } catch (Exception ex) { AddLog("Parser: " + ex.Message, LogMessageColor.Error()); } label6.SetPropertyThreadSafe(() => label6.Text, "Finish"); } }
private void OneLinkAvito() { try { label6.SetPropertyThreadSafe(() => label6.Text, "Start"); //Ссылка на обьявление mySqlDB.PrepareAvitoEnvironment(); //Создаем класс и вводим параметры var Parser = new RuslanParser(Properties.Default.User, Properties.Default.Password, Properties.Default.PathToProxy, mySqlDB); Parser.PathImages = Properties.Default.PathToImg; //вот тут мы вызывем запрос на Id к базе //Parser.LoadGuid = (() => MySqlDB.SeletGuid()); //Parser.LoadGuid = (() => "1532"); //тут мы присваем результат переменной result Dictionary<PartsPage, IEnumerable<string>> result = null; try { result = Parser.Run(URLLink); } catch (Exception ex) { AddLog("Parser: " + ex.Message, LogMessageColor.Error()); } // тут мы передаем в метод вставки данные //result, index ID, ссылку на обьявления //id я не вставлял так как непонятно было и неподходило под структуру бд if (result != null) { var idResourceList = mySqlDB.ResourceListIDAvito(); mySqlDB.InsertFctAvitoGrabber(result, idResourceList, URLLink, cbCategories.Text); var imageCount = new ImageParsedCountHelper(); imageCount.ResourceId = idResourceList; var Parser2 = new RuslanParser2(Properties.Default.User, Properties.Default.Password, Properties.Default.PathToProxy, mySqlDB, Properties.Default.FtpUsername, Properties.Default.FtpPassword, imageCount); Parser2.PathImages2 = Properties.Default.PathToImg; var result2 = Parser2.Run(URLLink); AddLog("Parser: " + imageCount.CountParsed + " images parsed", LogMessageColor.Information()); AddLog("Parser: " + imageCount.CountDownloaded + " images downloaded", LogMessageColor.Information()); foreach (var error in imageCount.ErrorList) { AddLog(error.Key, error.Value == true ? LogMessageColor.Error() : LogMessageColor.Success()); } foreach (var im in imageCount.Resources) mySqlDB.InsertassGrabberAvitoResourceList(idResourceList, im); mySqlDB.UpdateFctAvitoGrabberPhotoCount(idResourceList, imageCount.Resources.Count); AddLog("Parser: Loading images end", LogMessageColor.Information()); AddLog("Parser: Begin publishing ad", LogMessageColor.Information()); mySqlDB.ExecuteProcAvito(idResourceList); AddLog("Parser: End publishing ad", LogMessageColor.Information()); } } catch (Exception ex) { AddLog("Parser: " + ex.Message, LogMessageColor.Error()); } label6.SetPropertyThreadSafe(() => label6.Text, "Finish"); }