/// <summary>
/// 运行
/// </summary>
/// <param name="args">运行参数</param>
public static void Run(params string[] args)
{
var options = Parse(args);
if (options != null)
{
SetEncoding();
PrintEnviroment(args);
LoadConfiguration(options.Config);
var spiderName = options.Spider;
var spiderTypes = DetectSpiders();
if (spiderTypes == null || spiderTypes.Count == 0)
{
return;
}
var spider = CreateSpiderInstance(spiderName, options, spiderTypes);
if (spider != null)
{
PrintInfo.PrintLine();
var runMethod = spiderTypes[spiderName].GetMethod("Run");
runMethod.Invoke(spider, new object[] { options.Arguments });
}
}
}
private static void PrintEnviroment(params string[] args)
{
Console.WriteLine("");
PrintInfo.Print();
var commands = string.Join(" ", args);
Console.WriteLine($"Args : {commands}");
Console.WriteLine($"BaseDirectory : {AppDomain.CurrentDomain.BaseDirectory}");
Console.WriteLine($"System : {Environment.OSVersion} {(Environment.Is64BitOperatingSystem ? "X64" : "X86")}");
}
/// <summary>
/// 运行
/// </summary>
/// <param name="args">运行参数</param>
public static void Run(params string[] args)
{
SetConsoleEncoding();
PrintEnviroment(args);
Dictionary <string, string> arguments = AnalyzeArguments(args);
if (arguments == null || arguments.Count == 0)
{
return;
}
LoadConfiguration(arguments);
var spiderName = arguments["-s"];
var spiderTypes = DetectSpiders();
if (spiderTypes == null || spiderTypes.Count == 0)
{
return;
}
var spider = CreateSpiderInstance(spiderName, arguments, spiderTypes);
if (spider != null)
{
PrintInfo.PrintLine();
spiders.Enqueue(spider);
var runMethod = spiderTypes[spiderName].GetMethod("Run");
if (!arguments.ContainsKey("-a"))
{
runMethod.Invoke(spider, new object[] { new string[] { } });
}
else
{
var parameters = arguments["-a"].Split(new[] { ',' }, StringSplitOptions.RemoveEmptyEntries);
runMethod.Invoke(spider, new object[] { parameters });
}
}
}
public static Dictionary <string, Type> DetectSpiders()
{
var spiderTypes = new Dictionary <string, Type>();
foreach (var file in DetectDlls())
{
var asm = Assembly.Load(file);
var types = asm.GetTypes();
Console.WriteLine($"Fetch assembly: {asm.FullName}.");
foreach (var type in types)
{
bool hasNonParametersConstructor = type.GetConstructors().Any(c => c.IsPublic && c.GetParameters().Length == 0);
var fullName = type.FullName;
if (string.IsNullOrEmpty(fullName))
{
continue;
}
if (hasNonParametersConstructor)
{
var interfaces = type.GetInterfaces();
var isNamed = interfaces.Any(t => t.FullName == "DotnetSpider.Core.INamed");
var isIdentity = interfaces.Any(t => t.FullName == "DotnetSpider.Core.IIdentity");
var isRunnable = interfaces.Any(t => t.FullName == "DotnetSpider.Core.IRunable");
if (isNamed && isRunnable && isIdentity)
{
if (!spiderTypes.ContainsKey(fullName))
{
spiderTypes.Add(fullName, type);
}
else
{
Console.ForegroundColor = ConsoleColor.Red;
Console.WriteLine();
Console.WriteLine($"Spider {type.Name} are duplicate.");
Console.WriteLine();
Console.ForegroundColor = ConsoleColor.White;
return(null);
}
var startupName = type.GetCustomAttribute <SpiderName>();
if (startupName != null)
{
if (!spiderTypes.ContainsKey(startupName.Name))
{
spiderTypes.Add(startupName.Name, type);
}
else
{
Console.ForegroundColor = ConsoleColor.Red;
Console.WriteLine();
Console.WriteLine($"Spider {type.Name} are duplicate.");
Console.WriteLine();
Console.ForegroundColor = ConsoleColor.White;
return(null);
}
}
}
}
}
}
if (spiderTypes.Count == 0)
{
Console.ForegroundColor = ConsoleColor.DarkYellow;
Console.WriteLine();
Console.WriteLine("Did not detect any spider.");
Console.WriteLine();
Console.ForegroundColor = ConsoleColor.White;
return(null);
}
Console.WriteLine();
Console.ForegroundColor = ConsoleColor.Cyan;
Console.WriteLine($"Detected {spiderTypes.Keys.Count} crawlers.");
Console.ForegroundColor = ConsoleColor.White;
Console.WriteLine();
PrintInfo.PrintLine('=');
return(spiderTypes);
}