protected override void PreInitComponent(params string[] arguments)
{
base.PreInitComponent();
if (UseDbMonitor)
{
Monitor = new DbMonitor(Identity);
}
if (Site == null)
{
throw new SpiderException("Site should not be null.");
}
Scheduler.Init(this);
if (arguments.Contains("rerun"))
{
Scheduler.Clear();
Scheduler.Dispose();
BaseVerification.RemoveVerifidationLock(Identity);
}
}
public void Run(params string[] arguments)
{
if (string.IsNullOrEmpty(Identity) || Identity.Length > 120)
{
throw new ArgumentException("Length of Identity should between 1 and 120.");
}
if (string.IsNullOrEmpty(ConnectString))
{
ConnectString = Config.ConnectString;
}
if (string.IsNullOrEmpty(ConnectString))
{
throw new ArgumentException("ConnectString is missing.");
}
if (!string.IsNullOrEmpty(ConnectString))
{
NLogUtil.PrepareDatabase(ConnectString);
DbMonitor.InitStatusDatabase(ConnectString);
if (!string.IsNullOrEmpty(TaskId))
{
InsertRunningState();
}
using (IDbConnection conn = new MySqlConnection(ConnectString))
{
conn.Open();
var command = conn.CreateCommand();
command.CommandType = CommandType.Text;
command.CommandText = $"insert ignore into dotnetspider.status (`identity`, `status`,`thread`, `left`, `success`, `error`, `total`, `avgdownloadspeed`, `avgprocessorspeed`, `avgpipelinespeed`, `logged`) values('{Identity}', 'Init',-1, -1, -1, -1, -1, -1, -1, -1, '{DateTime.Now}');";
command.ExecuteNonQuery();
Logger.MyLog(Identity, $"开始任务: {Name}", LogLevel.Info);
}
_statusReporter = Task.Factory.StartNew(() =>
{
using (IDbConnection conn = new MySqlConnection(ConnectString))
{
conn.Open();
var command = conn.CreateCommand();
command.CommandType = CommandType.Text;
while (!_exited)
{
command.CommandText = $"update dotnetspider.status set `logged`='{DateTime.Now}' WHERE identity='{Identity}';";
command.ExecuteNonQuery();
Thread.Sleep(5000);
}
}
});
}
try
{
ImplementAction(arguments);
_exited = true;
_statusReporter.Wait();
Logger.MyLog(Identity, $"结束任务: {Name}", LogLevel.Info);
if (!string.IsNullOrEmpty(ConnectString))
{
using (IDbConnection conn = new MySqlConnection(ConnectString))
{
conn.Open();
var command = conn.CreateCommand();
command.CommandType = CommandType.Text;
command.CommandText = $"update dotnetspider.status set `status`='Finished',`logged`='{DateTime.Now}' WHERE identity='{Identity}';";
command.ExecuteNonQuery();
}
}
if (OnExited != null)
{
Verifier.ProcessVerifidation(Identity, OnExited);
}
}
catch (Exception e)
{
Logger.MyLog(Identity, $"退出任务: {Name}", LogLevel.Info);
if (!string.IsNullOrEmpty(ConnectString))
{
using (IDbConnection conn = new MySqlConnection(ConnectString))
{
conn.Open();
var command = conn.CreateCommand();
command.CommandType = CommandType.Text;
command.CommandText = $"update dotnetspider.status set `status`='Exited', `logged`='{DateTime.Now}' WHERE identity='{Identity}';";
command.ExecuteNonQuery();
}
}
}
finally
{
if (!string.IsNullOrEmpty(ConnectString) && !string.IsNullOrEmpty(TaskId))
{
RemoveRunningState();
}
}
}
protected override void PreInitComponent(params string[] arguments)
{
base.PreInitComponent();
if (UseDbLog)
{
Monitor = new DbMonitor(Identity);
}
if (Site == null)
{
throw new SpiderException("Site should not be null.");
}
if (Entities == null || Entities.Count == 0)
{
throw new SpiderException("Count of entity is zero.");
}
foreach (var entity in Entities)
{
foreach (var pipeline in Pipelines)
{
BaseEntityPipeline newPipeline = pipeline as BaseEntityPipeline;
newPipeline?.AddEntity(entity);
}
}
bool needInitStartRequest = true;
if (RedisConnection.Default != null)
{
if (arguments.Contains("rerun"))
{
RedisConnection.Default.Database.HashDelete(InitStatusSetKey, Identity);
RedisConnection.Default.Database.LockRelease(InitLockKey, "0");
}
while (!RedisConnection.Default.Database.LockTake(InitLockKey, "0", TimeSpan.FromMinutes(10)))
{
Thread.Sleep(1000);
}
var lockerValue = RedisConnection.Default.Database.HashGet(InitStatusSetKey, Identity);
needInitStartRequest = lockerValue != "init finished";
}
Scheduler.Init(this);
if (arguments.Contains("rerun"))
{
Scheduler.Clean();
Scheduler.Dispose();
Verifier.RemoveVerifidationLock(Identity);
needInitStartRequest = true;
}
if (needInitStartRequest && PrepareStartUrls != null)
{
for (int i = 0; i < PrepareStartUrls.Length; ++i)
{
var prepareStartUrl = PrepareStartUrls[i];
Logger.MyLog(Identity, $"[步骤 {i + 2}] 添加链接到调度中心.", LogLevel.Info);
prepareStartUrl.Build(this, null);
}
}
RegisterControl(this);
}
public void Run(params string[] arguments)
{
if (string.IsNullOrEmpty(Identity) || Identity.Length > 120)
{
throw new ArgumentException("Length of Identity should between 1 and 120.");
}
Monitor = new DbMonitor(Identity);
try
{
Logger.MyLog(Identity, $"Start: {Name}", LogLevel.Info);
if (Core.Environment.SystemConnectionStringSettings != null)
{
InsertRunningState();
_statusReporter = Task.Factory.StartNew(() =>
{
while (!_exited)
{
try
{
Monitor.Report("Running",
-1,
-1,
-1,
-1,
0,
0,
0,
-1);
}
catch (Exception e)
{
Logger.MyLog(Identity, $"Report status failed: {e}.", LogLevel.Error);
}
Thread.Sleep(5000);
}
});
}
if (arguments.Contains("rerun") || arguments.Contains("validate"))
{
BaseVerification.RemoveVerifidationLock(Identity);
}
ImplementAction(arguments);
_exited = true;
_statusReporter.Wait();
Logger.MyLog(Identity, $"Complete: {Name}", LogLevel.Info);
if (Core.Environment.SystemConnectionStringSettings != null)
{
try
{
Monitor.Report("Finished",
-1,
-1,
-1,
-1,
0,
0,
0,
-1);
}
catch (Exception e)
{
Logger.MyLog(Identity, $"Report status failed: {e}.", LogLevel.Error);
}
}
if (OnExited != null)
{
BaseVerification.ProcessVerifidation(Identity, OnExited);
}
}
catch (Exception e)
{
Logger.MyLog(Identity, $"Terminated: {Name}: {e}", LogLevel.Error, e);
if (Core.Environment.SystemConnectionStringSettings != null)
{
try
{
Monitor.Report("Terminated",
-1,
-1,
-1,
-1,
0,
0,
0,
-1);
}
catch (Exception e1)
{
Logger.MyLog(Identity, $"Report status failed: {e1}.", LogLevel.Error);
}
}
}
finally
{
if (Core.Environment.SystemConnectionStringSettings != null && !string.IsNullOrEmpty(TaskId))
{
RemoveRunningState();
}
}
}
