[TestMethod()] public void MultipleFilters()
{
var Logger = new LoggerProxy(Log);
var filters = Logger.Filters;
filters.Add(LogFilters.AddPrefix("Pre>"));
Log.WriteDelay = 0;
Log.Filters.Add(LogFilters.IgnoreDuplicateExceptions());
Log.Filters.Add(LogFilters.AutoMultiParam);
Log.Filters.Add(LogFilters.FormatParams);
Log.Filters.Add(LogFilters.ResolveLazy);
Log.Filters.Add(LogFilters.AddPostfix("<Post"));
Logger.Info("ARST");
Assert.IsTrue(LogContent.Contains("Pre>ARST"), "Simple Log");
Logger.Info("A", "R", null, "S", "T");
Assert.IsTrue(LogContent.Contains(">A R null S T<"), "Format multi params");
Logger.Info((Func <string>)(() => "QWFP"));
Assert.IsTrue(LogContent.Contains(">QWFP"), "Resolve lazy");
Exception subject = new Exception();
Logger.Error(subject);
Assert.IsTrue(LogContent.Contains("Pre>System.Exception:"), "Exception is logged");
var len = LogSize;
Logger.Error("Error: {0}", subject);
Assert.IsTrue(LogSize == len, "Duplicate is suppressed");
}
static void Main(string[] args)
{
System.AppDomain.CurrentDomain.UnhandledException += (sender, e) =>
{
var error = new StringBuilder("an unhandled exception occurred in the application.");
if (e?.ExceptionObject != null)
{
error.AppendFormat("\r\n{0}", e.ExceptionObject.ToString());
}
LoggerProxy.Error(LogSource, error.ToString());
};
TaskFactory.OnTaskRunning += (sender, e) => {
Console.WriteLine(string.Format("Task:{0},开始运行", e.Task.TaskName));
};
TaskFactory.OnTaskComplete += (sender, e) => {
Console.WriteLine(string.Format("Task:{0},结束运行，新增子任务数:{1}", e.Task.TaskName, e.Task.Stat.NewTaskCount));
};
var task = TaskFactory.InitTask("tasks/jd/task.js");
TaskFactory.Run(task.TaskId);
// TaskFactory.InitTasks("tasks");
// var tasks = TaskFactory.Tasks;
Console.ReadKey();
}
/// <summary>
/// 获取一个未执行的任务，并将此任务置为执行状态
/// </summary>
/// <param name="taskId">所属任务Id</param>
/// <returns></returns>
public TaskItemEntity GetExec(int taskId)
{
lock (this)
{
try
{
var filter = "{ \"taskId\" : " + taskId + " , \"status\" : { \"$in\" : [0,2] } }";
var taskInfo = this._db.SelectSingle <TaskItemEntity>(this._tbName, Serializer.JsonDeserialize(filter));
if (taskInfo != null)
{
taskInfo.Status = 1;
taskInfo.ExecCount++;
taskInfo.ExecTime = DateTime.Now;
lock (this)
{
return(this._db.Update(this._tbName, new { Id = taskInfo.Id }, taskInfo) == 1 ? taskInfo : null);
}
}
}
catch (Exception ex)
{
LoggerProxy.Error(LogSource, "call GetExec error.", ex);
}
return(null);
}
}
/// <summary>
/// 新增一个任务
/// </summary>
/// <param name="url">任务地址</param>
/// <param name="script">脚本文件名称</param>
public void newTask(string url, string script)
{
try
{
var invokerContext = ContextManager.GetTaskInvokerContext();
if (invokerContext == null)
{
throw new Exception("failed to get the taskInvokerContext.");
}
if (invokerContext.Task == null)
{
throw new Exception("failed to get the taskInvokerContext.Task.");
}
if (!TaskItems.Instance.AddObj(new
{
taskId = invokerContext.Task.TaskId,
parentId = invokerContext.TaskInvokerInfo?.Id,
url = url,
script = script
}))
{
LoggerProxy.Error(LogSource, string.Format("call newTask error, add task failed. url is '{0}',script is '{1}'.", url, script));
}
else
{
invokerContext.Task.SetStat(1, TaskStatTypes.NewTask);
}
}
catch (Exception ex)
{
LoggerProxy.Error(LogSource, string.Format("call newTask error. url is '{0}',script is '{1}'.", url, script), ex);
}
}
/// <summary>
/// 初始化脚本
/// </summary>
/// <param name="scriptFile"></param>
private bool InitScript(string scriptFile)
{
try
{
if (this._needInit)
{
// 初始化执行脚本
this._innerSE.Execute(FileUnity.ReadConfigFile(scriptFile));
this._innerSE.Execute(@"function ______tryRun(uri){
try{
var result = run(uri);
if(result == undefined || result == 1 || result == true){
return 'OK';
}
else{
return 'parse is not return 1 or true';
}
}catch(e){
return e.message;
}}");
this._needInit = false;
}
return(true);
}
catch (Exception ex)
{
LoggerProxy.Error(LogSource, string.Format("call InitScript error,scriptFile:'{0}'.{1}.", scriptFile, ex.Message), ex);
}
return(false);
}
/// <summary>
/// 开始下载
/// </summary>
public bool Down()
{
try
{
if (File.Exists(this._savePath))
{
return(true);
}
using (var localStream = GetLocalStream())
{
this._reqMsg.Headers.Range = new RangeHeaderValue(localStream.Length, null);
var res = this._http.SendAsync(this._reqMsg).ConfigureAwait(false).GetAwaiter().GetResult();
using (var remoteStream = res.Content.ReadAsStreamAsync().ConfigureAwait(false).GetAwaiter().GetResult())
{
remoteStream.CopyTo(localStream);
}
}
this.Compelte();
return(File.Exists(this._savePath));
}
catch (Exception ex)
{
LoggerProxy.Error(LogSource, string.Format("call Down error,url is '{0}', file is '{1}'.", this._reqMsg.RequestUri.ToString(), this._savePath), ex);
return(false);
}
}
/// <summary>
/// 初始化一个任务
/// </summary>
/// <param name="cfgFile">任务配置文件路径，如果已经初始化，则返回任务ID</param>
/// <returns>返回任务对象</returns>
public static Task InitTask(string cfgFile)
{
var fullPath = SnailCore.IO.PathUnity.GetFullPath(cfgFile);
if (string.IsNullOrEmpty(fullPath))
{
throw new Exception("could not find the file with path:" + cfgFile);
}
Task task = null;
lock (LockObj)
{
if (!BufferTasks.ContainsKey(fullPath))
{
task = new Task(fullPath);
foreach (var kv in BufferTasks)
{
if (kv.Value.TaskId == task.TaskId)
{
throw new Exception(string.Format("任务id:{0},重复出现,请检查配置,并重新启动任务.", task.TaskId));
}
}
task.OnStart += (sender, e) =>
{
try
{
OnTaskRunning?.Invoke(sender, e);
}
catch (Exception ex)
{
LoggerProxy.Error(LogSource, "invoke OnTaskRunning failed.", ex);
}
};
task.OnStop += (sender, e) =>
{
try
{
OnTaskComplete?.Invoke(sender, e);
}
catch (Exception ex)
{
LoggerProxy.Error(LogSource, "invoke OnTaskComplete failed.", ex);
}
};
task.OnError += (sender, e) =>
{
try
{
OnTaskError?.Invoke(sender, e);
}
catch (Exception ex)
{
LoggerProxy.Error(LogSource, "invoke OnTaskError failed.", ex);
}
};
BufferTasks.Add(fullPath, task);
}
}
return(BufferTasks[fullPath]);
}
/// <summary>
/// 释放资源
/// </summary>
private void Free()
{
// 释放任务执行者
if (this._freeSE.Count > 0)
{
do
{
try
{
if (this._freeSE.Count <= 0)
{
break;
}
var item = this._freeSE.Dequeue();
if (item == null)
{
break;
}
item.Dispose();
}
catch (Exception ex)
{
LoggerProxy.Error(LogSource, "call Free error.", ex);
}
}while (true);
}
}
/// <summary>
/// 运行脚本
/// </summary>
/// <param name="script"></param>
public bool Run(string script)
{
ContextManager.SetTaskInvokerContext(this.Context);
this._innerSE.Execute(script);
this._innerSE.Execute(@"function ______tryParse(){
try{
var result = parse();
if(result == undefined || result == 1 || result == true){
return 'OK';
}
else{
return 'parse is not return 1 or true';
}
}catch(e){
return e.message;
}}");
// 执行任务
var execRest = this._innerSE.Invoke("______tryParse");
if (execRest.ToString() == "OK")
{
return(true);
}
LoggerProxy.Error("taskTester", "执行任务失败," + execRest.ToString());
return(false);
}
/// <summary>
/// 加载指定的模块到JS运行环境中
/// </summary>
/// <param name="module">模块名称</param>
public object require(string module)
{
try
{
return(GetEngine()?.LoadModule(module));
}
catch (Exception ex)
{
LoggerProxy.Error(LogSource, string.Format("call require error.module name is '{0}'.", module), ex);
}
return(null);
}
private void dispose()
{
try
{
this._response?.RequestMessage?.Content?.Dispose();
this._response?.RequestMessage?.Dispose();
this._response?.Content?.Dispose();
this._response?.Dispose();
}
catch (Exception ex)
{
LoggerProxy.Error(LogSource, "dispose error.", ex);
}
}
public virtual HttpResult get(string uri, object data)
{
try
{
var res = this.SendAsync(this.NewHttpReqForPost(uri, data, HttpContentTypes.Json, HttpMethod.Get)).ConfigureAwait(false).GetAwaiter().GetResult();
return(new HttpResult(res));
}
catch (Exception ex)
{
LoggerProxy.Error(LogSource, "error", ex);
}
return(null);
}
public void config(dynamic cfg)
{
try
{
base.Config(new DbProviderConfig()
{
Driver = cfg.provider, Connection = cfg.connectionString
});
}
catch (Exception ex)
{
LoggerProxy.Error(LogSource, "call config error.", ex);
}
}
public virtual HttpResult get(string uri)
{
try
{
var res = this.SendAsync(this.NewHttpReqForGet(uri)).ConfigureAwait(false).GetAwaiter().GetResult();
return(new HttpResult(res));
}
catch (Exception ex)
{
LoggerProxy.Error(LogSource, "error", ex);
}
return(null);
}
/// <summary>
/// 按条件获取一个任务
/// </summary>
/// <param name="filter">过滤器</param>
/// <returns></returns>
public TaskItemEntity Get(object filter)
{
try
{
lock (this)
{
return(this._db.SelectSingle <TaskItemEntity>(this._tbName, filter));
}
}
catch (Exception ex)
{
LoggerProxy.Error(LogSource, "call Get error.", ex);
}
return(null);
}
/// <summary>
/// 添加一个任务
/// </summary>
/// <param name="task">任务配置{ url:xx,script:xx }</param>
/// <returns>返回是否添加成功</returns>
public bool AddObj(object task)
{
try
{
lock (this)
{
return(this._db.Insert(this._tbName, task) > 0);
}
}
catch (Exception ex)
{
LoggerProxy.Error(LogSource, "call AddObj error.", ex);
}
return(false);
}
/// <summary>
/// 执行脚本
/// </summary>
/// <param name="uri"></param>
/// <returns></returns>
private bool ExecItem(string uri)
{
try
{
var execRest = this._innerSE.Invoke("______tryRun", uri).ToString();
if (execRest != "OK")
{
throw new Exception(execRest);
}
return(true);
}
catch (Exception ex)
{
LoggerProxy.Error(LogSource, string.Format("call ExecItem error,url:'{0}'.{1}.", uri, ex.Message), ex);
}
return(false);
}
public bool add(string table, params object[] data)
{
try
{
var rest = base.insert(table, data);
var invokerContext = ContextManager.GetTaskInvokerContext();
if (rest > 0 && invokerContext != null && invokerContext.Task != null)
{
invokerContext.Task.SetStat(rest, TaskStatTypes.Article);
}
return(rest > 0);
}
catch (Exception ex)
{
LoggerProxy.Error(LogSource, string.Format("call add error,table is {0}.", table), ex);
}
return(false);
}
/// <summary>
/// 更新任务状态
/// </summary>
/// <param name="entity"></param>
/// <returns></returns>
public bool Update(TaskItemEntity entity)
{
try
{
lock (this)
{
return(this._db.Update(this._tbName, new
{
id = entity.Id
}, entity) == 1);
}
}
catch (Exception ex)
{
LoggerProxy.Error(LogSource, "call AddObj error.", ex);
}
return(false);
}
private static Assembly FindAssembly(string dir, string assFullName)
{
foreach (var file in new DirectoryInfo(dir).GetFiles("*.dll"))
{
try
{
var ass = Assembly.ReflectionOnlyLoadFrom(file.FullName);
if (ass.FullName == assFullName)
{
return(Assembly.LoadFile(file.FullName));
}
}
catch (Exception ex)
{
LoggerProxy.Error(LogSource, string.Format("call FindAssembly error.dir is '{0}', assName is '{1}'.", dir, assFullName), ex);
}
}
return(null);
}
/// <summary>
/// 获取一个完整路径
/// </summary>
/// <param name="uri"></param>
/// <param name="baseUri"></param>
/// <returns></returns>
public string getUri(string uri, string baseUri)
{
try
{
if (string.IsNullOrEmpty(baseUri))
{
baseUri = TaskInvokerContext.Current?.TaskInvokerInfo?.Url;
}
if (string.IsNullOrEmpty(baseUri))
{
return(uri);
}
return(new Uri(new Uri(baseUri), uri).ToString());
}
catch (Exception ex)
{
LoggerProxy.Error(LogSource, string.Format("call getUri error. url is '{0}',baseUri is '{1}'.", uri, baseUri), ex);
}
return("");
}
/// <summary>
/// 添加任务的最上层执行者,如果存在，不继续添加
/// </summary>
/// <param name="task"></param>
/// <returns></returns>
public bool AddRoot(TaskItemEntity task)
{
try
{
lock (this)
{
var taskInfo = this.Get(new { taskId = task.TaskId });
if (taskInfo != null)
{
return(true);
}
return(this.Add(task));
}
}
catch (Exception ex)
{
LoggerProxy.Error(LogSource, "call AddRoot error.", ex);
}
return(false);
}
public static bool getFile(this HttpModule http, params dynamic[] files)
{
try
{
List <dynamic> allFiles = new List <dynamic>();
var invokerContext = ContextManager.GetTaskInvokerContext();
if (invokerContext == null)
{
throw new Exception("failed to get the taskInvokerContext.");
}
foreach (var item in files)
{
if (item is JSArray)
{
allFiles.AddRange((JSArray)item);
}
else
{
allFiles.Add(item);
}
}
foreach (var file in allFiles)
{
if (!FileUnity.SafeCreateDirectory(file.savePath.Substring(0, file.savePath.LastIndexOf("\\"))))
{
return(false);
}
}
var result = http.getFiles(allFiles.ToArray());
if (result)
{
invokerContext.Task?.SetStat(files.Length, TaskStatTypes.File);
}
return(result);
}
catch (Exception ex)
{
LoggerProxy.Error("HttpModuleExtend", "call getFile error.", ex);
}
return(false);
}
/// <summary>
/// 添加一个任务
/// </summary>
/// <param name="task">任务对象</param>
/// <returns>返回是否添加成功</returns>
public bool Add(TaskItemEntity task)
{
try
{
lock (this)
{
return(this._db.Insert(this._tbName, new
{
parentId = task.ParentId,
script = task.Script,
taskId = task.TaskId,
url = task.Url,
}) > 0);
}
}
catch (Exception ex)
{
LoggerProxy.Error(LogSource, "call Add error.", ex);
}
return(false);
}
/// <summary>
/// 获取一个新的采集任务，如果当前繁忙，此方法将会等待
/// </summary>
/// <returns></returns>
private TaskInvoker NewInvoker()
{
try
{
var item = TaskItems.Instance.GetExec(this.TaskId);
if (item == null)
{
return(null);
}
var settings = new TaskItemSetting()
{
ScriptFile = System.IO.Path.Combine(this.ExecutePath, item.Script),
Url = item.Url,
TaskInvokerInfo = item
};
this._lock.WaitOne();
TaskInvoker invoker = null;
lock (this)
{
if (this._freeSE.Count > 0)
{
invoker = this._freeSE.Dequeue();
}
if (invoker == null)
{
invoker = new TaskInvoker(this);
this._count++;
}
}
invoker.SetSetting(settings);
Interlocked.Increment(ref this._busyCount);
return(invoker);
}
catch (Exception ex)
{
LoggerProxy.Error(LogSource, "call NewInvoker failed.", ex);
}
return(null);
}
public string toString(bool autoDispose = true)
{
try
{
if (this._response == null)
{
return(null);
}
var result = this._response.Content.ReadAsStringAsync().Result;
if (autoDispose)
{
this._response.Dispose();
this._response = null;
}
return(result);
}
catch (Exception ex)
{
LoggerProxy.Error(LogSource, "error", ex);
}
return(null);
}
/// <summary>
/// 工作线程执行方法
/// </summary>
private void ThreadWork()
{
this.BindContext();
do
{
this._notify.WaitOne();
if (!this.InitScript(this.CurrSetting.ScriptFile))
{
continue;
}
this.Result.Success = ExecItem(this.CurrSetting.Url);
this.Status = TaskInvokerStatus.Stop;
try
{
this._callBack?.Invoke(this);
}
catch (Exception ex)
{
LoggerProxy.Error(LogSource, string.Format("call ThreadWork-item callback error,url:'{0}'.", this.CurrSetting.Url), ex);
}
}while (true);
}
/// <summary>
/// 释放一个任务执行者
/// </summary>
/// <param name="invoker"></param>
private void FreeInvoker(TaskInvoker invoker)
{
try
{
if (invoker.Result != null)
{
// 设置任务状态
if (invoker.Result.TaskInvokerInfo != null)
{
invoker.Result.TaskInvokerInfo.Status = invoker.Result.Success ? 3 :
invoker.Result.TaskInvokerInfo.ExecCount >= ConfigManager.Current.ErrorRetry ? 4 : 2;
if (!TaskItems.Instance.Update(invoker.Result.TaskInvokerInfo))
{
LoggerProxy.Error(LogSource, "update callInvoker status failed.");
}
// 新增统计信息
this.SetStat(1, invoker.Result.Success ? TaskStatTypes.Task : TaskStatTypes.ErrTask);
}
}
}
catch (Exception ex)
{
LoggerProxy.Error(LogSource, "call FreeInvoker failed.", ex);
}
finally
{
try
{
Interlocked.Decrement(ref this._busyCount);
lock (this)
{
this._freeSE.Enqueue(invoker);
this._lock.Release();
}
}
catch { }
}
}
/// <summary>
/// 执行任务
/// </summary>
internal bool Run()
{
Init();
if (Status != TaskStatus.Init)
{
return(false);
}
lock (LockObj)
{
if (Status != TaskStatus.Init)
{
return(false);
}
Status = TaskStatus.Running;
}
new Thread(() =>
{
try
{
while (true)
{
var isStop = false;
lock (this)
{
isStop = this.Status == TaskStatus.Stopping;
}
// 获取执行者
TaskInvoker invoker = isStop ? null : NewInvoker();
if (invoker == null)
{
if (SetStop())
{
OnStop?.Invoke(this, new TaskEventArgs()
{
Task = this
});
this.Free();
break;
}
Thread.Sleep(1000);
continue;
}
invoker.RunAsync((task) =>
{
// 释放执行者
FreeInvoker(task);
});
}
}
catch (Exception ex)
{
LoggerProxy.Error(LogSource, "call Run error.", ex);
OnError?.Invoke(this, new TaskErrorEventArgs()
{
Ex = ex,
Task = this
});
}
}).Start();
OnStart?.Invoke(this, new TaskEventArgs()
{
Task = this
});
return(true);
}