public async void execute()
{
Console.WriteLine("Запуск процесса зборщика слова по URL");
List <Task> taskList = new List <Task>();
while (true)
{
kwsiteMutext.WaitOne();
var globalOffset = !settings.KeyExists("offsetSite") ?
Int32.Parse(settings.Write("offsetSite", "0")) :
Int32.Parse(settings.Read("offsetSite"));
this.offsetSite = Math.Max(this.offsetSite, globalOffset);
this.limitSite = (STEP - Math.Min(taskList.Count, STEP));
this.offsetSite += (STEP - Math.Min(taskList.Count, STEP));
globalOffset += this.limitSite;
settings.Write("offsetSite", "" + this.offsetSite);
List <String> list = this.listSites(offsetSite, limitSite);
if (list == null)
{
kwsiteMutext.ReleaseMutex();
break;
}
kwsiteMutext.ReleaseMutex();
list.ForEach(site =>
{
taskList.Add(Task.Run(() =>
{
this.takeKwBySite(site);
}));
Thread.Sleep(YandexUtils.rndSleep());
});
Console.WriteLine("Задач в таске:" + taskList.Count());
if (taskList.Count >= this.limitSite)
{
Task.WaitAny(taskList.ToArray());
try
{
taskList
.Where(t => t.IsCompleted)
.ToList()
.ForEach(t =>
{
taskList.Remove(t);
});
}
catch { }
}
Thread.Sleep(YandexUtils.rndSleep());
}
}
public ForecastGetherer()
{
settings = new IniFiles(System.IO.Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location) + "/Settings.ini");
this.forecastOffset = !settings.KeyExists("offset", SETTINGS_SECTION) ?
Int64.Parse(settings.Write("offset", "0", SETTINGS_SECTION)) :
Int64.Parse(settings.Read("offset", SETTINGS_SECTION));
}
private void init()
{
this.clearCSV();
this.offsetSite = 0;
this.limitSite = STEP;
settings = new IniFiles(System.IO.Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location) + "/Settings.ini");
this.offsetSite = !settings.KeyExists("offsetSite") ?
Int32.Parse(settings.Write("offsetSite", "0")) :
Int32.Parse(settings.Read("offsetSite"));
this.limitSite = !settings.KeyExists("limitSite") ?
Int32.Parse(settings.Write("limitSite", "" + STEP)) :
Int32.Parse(settings.Read("limitSite"));
}
private void init()
{
this.clearCSV();
this.offset = 0;
this.limit = STEP;
this.file_offset = 0;
this.loadFromFile = false;
this.loadFromPath = "";
settings = new IniFiles(System.IO.Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location) + "/Settings.ini");
this.limit = !settings.KeyExists("limit", SETTINGS_SECTION) ?
Int32.Parse(settings.Write("limit", "" + STEP, SETTINGS_SECTION)) :
Int32.Parse(settings.Read("limit", SETTINGS_SECTION));
this.loadFromFile = !settings.KeyExists("loadFromFile", SETTINGS_SECTION) ?
Boolean.Parse(settings.Write("loadFromFile", "false", SETTINGS_SECTION)) :
Boolean.Parse(settings.Read("loadFromFile", SETTINGS_SECTION));
this.loadFromPath = System.IO.Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location) + "/"
+ (!settings.KeyExists("loadFromPath", SETTINGS_SECTION) ?
settings.Write("loadFromPath", "words.txt", SETTINGS_SECTION) :
settings.Read("loadFromPath", SETTINGS_SECTION));
if (loadFromFile)
{
this.offset = !settings.KeyExists("file_offset", SETTINGS_SECTION) ?
long.Parse(settings.Write("file_offset", "" + file_offset, SETTINGS_SECTION)) :
long.Parse(settings.Read("file_offset", SETTINGS_SECTION));
}
else
{
this.offset = !settings.KeyExists("offset", SETTINGS_SECTION) ?
long.Parse(settings.Write("offset", "" + offset, SETTINGS_SECTION)) :
long.Parse(settings.Read("offset", SETTINGS_SECTION));
}
}
private void init()
{
this.clearCSV();
this.offset = 0;
this.limit = STEP;
this.file_offset = 0;
this.loadFromFile = false;
this.loadFromPath = "";
settings = new IniFiles("Settings.ini");
this.limit = !settings.KeyExists("limit") ?
Int32.Parse(settings.Write("limit", "" + STEP)) :
Int32.Parse(settings.Read("limit"));
this.loadFromFile = !settings.KeyExists("loadFromFile") ?
Boolean.Parse(settings.Write("loadFromFile", "false")) :
Boolean.Parse(settings.Read("loadFromFile"));
this.loadFromPath = !settings.KeyExists("loadFromPath") ?
settings.Write("loadFromPath", "words.txt") :
settings.Read("loadFromPath");
if (loadFromFile)
{
this.offset = !settings.KeyExists("file_offset") ?
Int32.Parse(settings.Write("file_offset", "" + file_offset)) :
Int32.Parse(settings.Read("file_offset"));
}
else
{
this.offset = !settings.KeyExists("offset") ?
Int32.Parse(settings.Write("offset", "" + offset)) :
Int32.Parse(settings.Read("offset"));
}
}
public async void execute()
{
File.ReadAllLines(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location) + "/accounts.txt")
.ToList()
.ForEach(s =>
{
string[] buf = s.Split(';');
Console.WriteLine("Выбираем аккаунт Login [{0}], AppId [{1}], Token [{2}]", buf[0], buf[1], buf[2]);
Task.Run(() =>
{
YandexDirect yandexDirect = new YandexDirect(buf[0], buf[1], buf[2]);
while (true)
{
forecastMutext.WaitOne();
var globalOffset = !settings.KeyExists("offset", SETTINGS_SECTION) ?
Int64.Parse(settings.Write("offset", "0", SETTINGS_SECTION)) :
Int64.Parse(settings.Read("offset", SETTINGS_SECTION));
this.forecastOffset = Math.Max(this.forecastOffset, globalOffset);
Console.WriteLine("Take date from {0} pos to {1}", this.forecastOffset, this.forecastOffset + LIMIT);
List <string> temp = new List <string>();
List <DBKeyword> tempList = this.listKeywords(this.forecastOffset, LIMIT);
if (tempList.Count == 0 || tempList == null)
{
break;
}
this.forecastOffset += LIMIT;
globalOffset += LIMIT;
settings.Write("offset", "" + Math.Max(this.forecastOffset, globalOffset), SETTINGS_SECTION);
forecastMutext.ReleaseMutex();
tempList
.ForEach(w =>
{
temp.Add(w.keyword);
temp.Add(w.keyword.checkLenAndSlice().divideAndPrecede());
});
Console.WriteLine("Запускаем отчет");
yandexDirect.execute(temp.ToArray());
}
});
});
}
public async void execute()
{
Console.WriteLine("Запуск процесса зборщика слова");
List <Task> taskList = new List <Task>();
long count = this.fileDataCount();
while (true)
{
kwMutext.WaitOne();
var globalOffset = this.offset;
if (loadFromFile)
{
globalOffset = !settings.KeyExists("file_offset", SETTINGS_SECTION) ?
long.Parse(settings.Write("file_offset", "" + file_offset, SETTINGS_SECTION)) :
long.Parse(settings.Read("file_offset", SETTINGS_SECTION));
}
else
{
globalOffset = !settings.KeyExists("offset", SETTINGS_SECTION) ?
long.Parse(settings.Write("offset", "" + offset, SETTINGS_SECTION)) :
long.Parse(settings.Read("offset", SETTINGS_SECTION));
}
this.offset = Math.Max(globalOffset, this.offset);
this.limit = (STEP - Math.Min(taskList.Count, STEP));
this.offset += (STEP - Math.Min(taskList.Count, STEP));
settings.Write(this.loadFromFile ? "file_offset" : "offset", "" + this.offset, SETTINGS_SECTION);
List <DBKeyword> list = loadFromFile ? loadFile(offset, limit) : this.listKeywords(offset, limit);
if (list == null)
{
kwMutext.ReleaseMutex();
break;
}
kwMutext.ReleaseMutex();
list.ForEach(kw =>
{
taskList.Add(Task.Run(() =>
{
this.takeKW(kw);
}));
Thread.Sleep(YandexUtils.rndSleep());
});
Console.WriteLine("Задач в таске:" + taskList.Count());
Task.WaitAny(taskList.ToArray());
try
{
taskList
.Where(t => t.IsCompleted)
.ToList()
.ForEach(t => {
taskList.Remove(t);
});
}
catch {
}
if (loadFromFile && offset >= count)
{
settings.Write("loadFromFile", "false", SETTINGS_SECTION);
offset = long.Parse(settings.Read("offset", SETTINGS_SECTION));
loadFromFile = false;
Console.WriteLine("Переключаемся на выборку из БД");
Thread.Sleep(YandexUtils.rndSleep());
}
Thread.Sleep(YandexUtils.rndSleep());
Console.WriteLine("offset=>" + offset + " count=>" + count);
}
}
