public async void execute() { while (true) { List <DBKeyword> list = loadFromFile ? loadFile(offset, limit) : this.listKeywords(offset, limit); int count = list.Count(); List <Task> taskList = new List <Task>(); list.ForEach(kw => { var task = Task.Run(() => this.takeKW(kw)); taskList.Add(task); }); Console.WriteLine("Задач в таске:" + taskList.Count()); Task.WaitAll(taskList.ToArray()); this.offset += STEP; settings.Write(this.loadFromFile ? "file_offset" : "offset", "" + this.offset); if (loadFromFile && offset >= count) { settings.Write("loadFromFile", "false"); offset = Int32.Parse(settings.Read("offset")); loadFromFile = false; count = this.countKewyrods(); Console.WriteLine("Переключаемся на выборку из БД"); } if (offset >= count) { break; } } }
public async void execute() { Console.WriteLine("Запуск процесса зборщика слова по URL"); List <Task> taskList = new List <Task>(); while (true) { kwsiteMutext.WaitOne(); var globalOffset = !settings.KeyExists("offsetSite") ? Int32.Parse(settings.Write("offsetSite", "0")) : Int32.Parse(settings.Read("offsetSite")); this.offsetSite = Math.Max(this.offsetSite, globalOffset); this.limitSite = (STEP - Math.Min(taskList.Count, STEP)); this.offsetSite += (STEP - Math.Min(taskList.Count, STEP)); globalOffset += this.limitSite; settings.Write("offsetSite", "" + this.offsetSite); List <String> list = this.listSites(offsetSite, limitSite); if (list == null) { kwsiteMutext.ReleaseMutex(); break; } kwsiteMutext.ReleaseMutex(); list.ForEach(site => { taskList.Add(Task.Run(() => { this.takeKwBySite(site); })); Thread.Sleep(YandexUtils.rndSleep()); }); Console.WriteLine("Задач в таске:" + taskList.Count()); if (taskList.Count >= this.limitSite) { Task.WaitAny(taskList.ToArray()); try { taskList .Where(t => t.IsCompleted) .ToList() .ForEach(t => { taskList.Remove(t); }); } catch { } } Thread.Sleep(YandexUtils.rndSleep()); } }
public async void execute() { Console.WriteLine("Запуск процесса зборщика слова"); List <Task> taskList = new List <Task>(); long count = this.fileDataCount(); while (true) { kwMutext.WaitOne(); this.limit = (STEP - Math.Min(taskList.Count, STEP)); this.offset += (STEP - Math.Min(taskList.Count, STEP)); settings.Write(this.loadFromFile ? "file_offset" : "offset", "" + this.offset, SETTINGS_SECTION); List <DBKeyword> list = loadFromFile ? loadFile(offset, limit) : this.listKeywords(offset, limit); if (list == null) { kwMutext.ReleaseMutex(); break; } kwMutext.ReleaseMutex(); list.ForEach(kw => { taskList.Add(Task.Run(() => { this.takeKW(kw); })); Thread.Sleep(YandexUtils.rndSleep()); }); Console.WriteLine("Задач в таске:" + taskList.Count()); Task.WaitAny(taskList.ToArray()); try { taskList .Where(t => t.IsCompleted) .ToList() .ForEach(t => { taskList.Remove(t); }); } catch { } if (loadFromFile && offset >= count) { settings.Write("loadFromFile", "false", SETTINGS_SECTION); offset = long.Parse(settings.Read("offset", SETTINGS_SECTION)); loadFromFile = false; Console.WriteLine("Переключаемся на выборку из БД"); Thread.Sleep(YandexUtils.rndSleep()); } Thread.Sleep(YandexUtils.rndSleep()); Console.WriteLine("offset=>" + offset + " count=>" + count); } }
public ForecastGetherer() { settings = new IniFiles(System.IO.Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location) + "/Settings.ini"); this.forecastOffset = !settings.KeyExists("offset", SETTINGS_SECTION) ? Int64.Parse(settings.Write("offset", "0", SETTINGS_SECTION)) : Int64.Parse(settings.Read("offset", SETTINGS_SECTION)); }
private void init() { this.clearCSV(); this.offsetSite = 0; this.limitSite = STEP; settings = new IniFiles(System.IO.Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location) + "/Settings.ini"); this.offsetSite = !settings.KeyExists("offsetSite") ? Int32.Parse(settings.Write("offsetSite", "0")) : Int32.Parse(settings.Read("offsetSite")); this.limitSite = !settings.KeyExists("limitSite") ? Int32.Parse(settings.Write("limitSite", "" + STEP)) : Int32.Parse(settings.Read("limitSite")); }
private void init() { this.clearCSV(); this.offset = 0; this.limit = STEP; this.file_offset = 0; this.loadFromFile = false; this.loadFromPath = ""; settings = new IniFiles(System.IO.Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location) + "/Settings.ini"); this.limit = !settings.KeyExists("limit", SETTINGS_SECTION) ? Int32.Parse(settings.Write("limit", "" + STEP, SETTINGS_SECTION)) : Int32.Parse(settings.Read("limit", SETTINGS_SECTION)); this.loadFromFile = !settings.KeyExists("loadFromFile", SETTINGS_SECTION) ? Boolean.Parse(settings.Write("loadFromFile", "false", SETTINGS_SECTION)) : Boolean.Parse(settings.Read("loadFromFile", SETTINGS_SECTION)); this.loadFromPath = System.IO.Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location) + "/" + (!settings.KeyExists("loadFromPath", SETTINGS_SECTION) ? settings.Write("loadFromPath", "words.txt", SETTINGS_SECTION) : settings.Read("loadFromPath", SETTINGS_SECTION)); if (loadFromFile) { this.offset = !settings.KeyExists("file_offset", SETTINGS_SECTION) ? long.Parse(settings.Write("file_offset", "" + file_offset, SETTINGS_SECTION)) : long.Parse(settings.Read("file_offset", SETTINGS_SECTION)); } else { this.offset = !settings.KeyExists("offset", SETTINGS_SECTION) ? long.Parse(settings.Write("offset", "" + offset, SETTINGS_SECTION)) : long.Parse(settings.Read("offset", SETTINGS_SECTION)); } }
private void init() { this.clearCSV(); this.offset = 0; this.limit = STEP; this.file_offset = 0; this.loadFromFile = false; this.loadFromPath = ""; settings = new IniFiles("Settings.ini"); this.limit = !settings.KeyExists("limit") ? Int32.Parse(settings.Write("limit", "" + STEP)) : Int32.Parse(settings.Read("limit")); this.loadFromFile = !settings.KeyExists("loadFromFile") ? Boolean.Parse(settings.Write("loadFromFile", "false")) : Boolean.Parse(settings.Read("loadFromFile")); this.loadFromPath = !settings.KeyExists("loadFromPath") ? settings.Write("loadFromPath", "words.txt") : settings.Read("loadFromPath"); if (loadFromFile) { this.offset = !settings.KeyExists("file_offset") ? Int32.Parse(settings.Write("file_offset", "" + file_offset)) : Int32.Parse(settings.Read("file_offset")); } else { this.offset = !settings.KeyExists("offset") ? Int32.Parse(settings.Write("offset", "" + offset)) : Int32.Parse(settings.Read("offset")); } }
public async void execute() { File.ReadAllLines(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location) + "/accounts.txt") .ToList() .ForEach(s => { string[] buf = s.Split(';'); Console.WriteLine("Выбираем аккаунт Login [{0}], AppId [{1}], Token [{2}]", buf[0], buf[1], buf[2]); Task.Run(() => { YandexDirect yandexDirect = new YandexDirect(buf[0], buf[1], buf[2]); while (true) { forecastMutext.WaitOne(); var globalOffset = !settings.KeyExists("offset", SETTINGS_SECTION) ? Int64.Parse(settings.Write("offset", "0", SETTINGS_SECTION)) : Int64.Parse(settings.Read("offset", SETTINGS_SECTION)); this.forecastOffset = Math.Max(this.forecastOffset, globalOffset); Console.WriteLine("Take date from {0} pos to {1}", this.forecastOffset, this.forecastOffset + LIMIT); List <string> temp = new List <string>(); List <DBKeyword> tempList = this.listKeywords(this.forecastOffset, LIMIT); if (tempList.Count == 0 || tempList == null) { break; } this.forecastOffset += LIMIT; globalOffset += LIMIT; settings.Write("offset", "" + Math.Max(this.forecastOffset, globalOffset), SETTINGS_SECTION); forecastMutext.ReleaseMutex(); tempList .ForEach(w => { temp.Add(w.keyword); temp.Add(w.keyword.checkLenAndSlice().divideAndPrecede()); }); Console.WriteLine("Запускаем отчет"); yandexDirect.execute(temp.ToArray()); } }); }); }