public void ArchivePrevMonth()
{
var dt = DateTime.Now.AddMonths(-1);
DateTime startDate = new DateTime(dt.Year, dt.Month, 1);
while (startDate < DateTime.Now)
{
DateTime endDate = startDate.AddMonths(1).AddSeconds(-1);
DBHelpers.GetPages(ProcessPage, null, startDate, endDate);
startDate = startDate.AddMonths(1);
break;
}
}
public void Run()
{
//IndexPrevMonth();
//ArchivePrevMonth();
//return;
//index files
//IndexFiles();
//index current period
ThreadPool.QueueUserWorkItem(x =>
{
var startDate = new DateTime(DateTime.Now.Year, DateTime.Now.Month, 1);
DBHelpers.GetPages(IndexFile, null, startDate, startDate.AddMonths(1));
});
Thread.Sleep(500);
this.IndexEqueue();
try
{
//run spiders
foreach (Site site in BH.BoobenRobot.Site.Sites)
{
//site.LoadLabels();
ThreadPool.QueueUserWorkItem(new WaitCallback(site.Run));
}
//refresh interface
UpdateStat();
ThreadPool.QueueUserWorkItem(new WaitCallback(this.UpdateStat));
}
catch (Exception ex)
{
_logError(ex.Message);
}
}
private void IndexFiles()
{
var info = _service.GetInfo();
if (info.LastNameIDRAM <= 80) //empty instance
{
/*
* string logPath = BH.BoobenRobot.Site.FTRobot_PATH + "Log.txt";
*
* if (File.Exists(logPath))
* {
* StreamReader sr = new StreamReader(logPath);
*
* //read files
* List<string> files = new List<string>();
*
* while (!sr.EndOfStream)
* {
* string file = sr.ReadLine();
* files.Add(file);
* }
*
* sr.Close();
*
* //distinct files
* List<string> disFiles = new List<string>();
*
* Dictionary<string, bool> dic = new Dictionary<string, bool>();
*
* for (int i = files.Count - 1; i >= 0; i--)
* {
* string file = files[i];
*
* if (!dic.ContainsKey(file))
* {
* disFiles.Insert(0, file);
*
* dic.Add(file, true);
* }
* }
*
* StringBuilder log = new StringBuilder();
*
* //index files
* int count = 1;
* foreach (string file1 in disFiles)
* {
* string file = file1.Replace("ixbt-", "ixbt_");
*
* if (File.Exists(file))
* {
* string aliasName = Path.GetFileName(file).Replace(".txt", "");
* string contentText = File.ReadAllText(file, BH.BoobenRobot.Site.ASC2Encoding);
*
* //string[] parts = contentText.Split(new char[] { '.', '?', '!' });
* //foreach (string part in parts)
* //{
* // if (part.Trim().Length > 3 && part.Trim().Length < 65535)
* // {
*
* if (contentText.Length > 65000)
* {
* contentText = contentText.Substring(0, 65000);
* }
*
* client.IndexText(aliasName, contentText);
*
* // }
* //}
*
* if (count % 100 == 0)
* {
* tbLog.Text = "Index files: " + count.ToString() + " of " + disFiles.Count.ToString();
* tbLog.Update();
*
* Application.DoEvents();
* }
*
* count++;
*
* //save new log
* log.AppendLine(file);
* }
* }
*
* //save log
* File.WriteAllText(logPath, log.ToString());
* }
*/
//index previous periods
DateTime startDate;
ThreadPool.QueueUserWorkItem(x =>
{
startDate = new DateTime(2015, 6, 1);
while (true)
{
string zipPath;
for (int archiveIndex = 1; ; archiveIndex++)
{
if (archiveIndex == 1)
{
zipPath = Site.FTRobot_PATH + startDate.Year.ToString("0000") + startDate.Month.ToString("00") + ".zip";
}
else
{
zipPath = Site.FTRobot_PATH + startDate.Year.ToString("0000") + startDate.Month.ToString("00") + "_" + archiveIndex.ToString() + ".zip";
}
if (File.Exists(zipPath))
{
using (FileStream zipToOpen = new FileStream(zipPath, FileMode.Open))
{
using (ZipArchive archive = new ZipArchive(zipToOpen, ZipArchiveMode.Read))
{
DBHelpers.GetPages(IndexFile, archive, startDate, startDate.AddMonths(1));
}
}
}
else
{
break;
}
}
if (startDate < new DateTime(DateTime.Now.Year, DateTime.Now.Month, 1))
{
startDate = startDate.AddMonths(1);
}
else
{
break;
}
}
});
Thread.Sleep(100);
IndexEqueue();
////index current period
//startDate = new DateTime(DateTime.Now.Year, DateTime.Now.Month, 1);
//DBHelpers.GetPages(IndexFile, null, startDate, startDate.AddMonths(1));
//save index
//client.SaveIndex();
}
}