/// <summary>
/// 添加一个采集执行者，当该页面不存在时
/// </summary>
/// <param name="collectTaskInvoker"></param>
/// <returns></returns>
public bool AddWhereNotExists(CollectTaskInvoker collectTaskInvoker)
{
return(this._lock.SafeSetValue(item =>
{
var uri = item.Uri.ToString();
if (!this._dic.ContainsKey(uri))
{
this._dic.Add(uri, collectTaskInvoker);
return true;
}
return false;
}, collectTaskInvoker));
}
/// <summary>
/// 开始采集
/// </summary>
/// <param name="cancellationToken"></param>
/// <param name="collectInfo"></param>
public void Start(CancellationToken cancellationToken, CollectInfo collectInfo)
{
this._collect = collectInfo;
// 重置任务状态
this._collectTaskDal.Update(collectInfo.Id, CollectTaskStatus.Running, CollectTaskStatus.None);
if (cancellationToken.IsCancellationRequested)
{
return;
}
// 执行初始化脚本，不做异常捕获，发生异常直接退出
using (var initInvoker = new CollectTaskInvoker(this.State))
{
initInvoker.Invoke(collectInfo, null);
}
if (cancellationToken.IsCancellationRequested)
{
return;
}
// 循环执行
while (true)
{
// 取消执行，等待其他任务结束
if (cancellationToken.IsCancellationRequested)
{
if (this.State.RunningTaskCount > 0)
{
Thread.Sleep(500);
continue;
}
return;
}
// 请求执行
this._sh.WaitOne();
var taskInfo = this._collectTaskDal.SelectSingle(collectInfo.Id, CollectTaskStatus.None);
if (taskInfo == null)
{
taskInfo = this._collectTaskDal.SelectSingle(collectInfo.Id, CollectTaskStatus.Error);
}
// 获取不到，等待其他任务结束
if (taskInfo == null)
{
if (this.State.RunningTaskCount > 0)
{
Thread.Sleep(500);
this._sh.Release();
continue;
}
return;
}
taskInfo.Status = CollectTaskStatus.Running;
taskInfo.RetryCount++;
this._collectTaskDal.Update(taskInfo);
// 发起新线程，执行任务
Task.Factory.StartNew((objTask) =>
{
this.State.RunningTaskCount = 1;
CollectTaskInvoker invoker = null;
var refTask = objTask as CollectTaskInfo;
var invokeArgs = new CollectTaskInvokeCompleteArgs()
{
Task = refTask
};
try
{
invoker = this._lock.SafeReadValue(() =>
{
if (this._invokers.Count > 0)
{
return(this._invokers.Dequeue());
}
return(new CollectTaskInvoker(this.State));
});
invoker.Invoke(this._collect, refTask);
refTask.Status = CollectTaskStatus.Complete;
this._collectTaskDal.Update(refTask);
invokeArgs.Success = true;
}
catch (Exception ex)
{
refTask.Status = CollectTaskStatus.Error;
if (refTask.RetryCount >= this._retry)
{
refTask.Status = CollectTaskStatus.Faild;
}
this._collectTaskDal.Update(refTask);
invokeArgs.Success = false;
invokeArgs.Error = ex;
}
finally
{
// 单个任务执行结束
if (invoker != null)
{
this._lock.SafeSetValue(t => this._invokers.Enqueue(t), invoker);
}
if (invokeArgs.Success)
{
this.State.CompleteTaskCount = 1;
}
else
{
this.State.ErrorTaskCount = 1;
}
this.OnCollectTaskInvokeComplete?.Invoke(this, invokeArgs);
this.State.RunningTaskCount = -1;
this._sh.Release();
}
}, taskInfo);
}
}
