/// <summary>
/// 运行
/// </summary>
/// <param name="args">运行参数</param>
public void Execute(params string[] args)
{
try
{
ConfigureSerialLog();
Framework.SetEncoding();
Framework.SetMultiThread();
var configurationBuilder = new ConfigurationBuilder();
configurationBuilder.SetBasePath(AppDomain.CurrentDomain.BaseDirectory);
configurationBuilder.AddEnvironmentVariables();
configurationBuilder.AddCommandLine(Environment.GetCommandLineArgs(), Framework.SwitchMappings);
var configuration = configurationBuilder.Build();
string spiderTypeName = configuration["type"];
if (string.IsNullOrWhiteSpace(spiderTypeName))
{
Log.Logger.Error("未指定需要执行的爬虫类型");
return;
}
var name = configuration["name"];
var id = configuration["id"] ?? Guid.NewGuid().ToString("N");
var arguments = configuration["args"]?.Split(' ');
PrintEnvironment();
var spiderTypes = DetectSpiders();
if (spiderTypes == null || spiderTypes.Count == 0)
{
return;
}
var spiderType = spiderTypes.FirstOrDefault(x => x.UnderlyingSystemType.ToString().ToLower() == spiderTypeName.ToLower());
if (spiderType == null)
{
Log.Logger.Error($"未找到爬虫: {spiderTypeName}", 0, ConsoleColor.DarkYellow);
return;
}
var builder = new SpiderHostBuilder();
ConfigureService(configuration, builder);
builder.Register(spiderType);
var provider = builder.Build();
var instance = provider.Create(spiderType);
if (instance != null)
{
instance.Name = name;
instance.Id = id;
instance.RunAsync(arguments).ConfigureAwait(true).GetAwaiter().GetResult();
}
else
{
Log.Logger.Error("创建爬虫对象失败", 0, ConsoleColor.DarkYellow);
}
}
catch (Exception e)
{
Log.Logger.Error($"执行失败: {e}");
}
}
public static void Execute <TSpider>(params string[] args)
{
var logfile = Environment.GetEnvironmentVariable("DOTNET_SPIDER_ID");
logfile = string.IsNullOrWhiteSpace(logfile) ? "dotnet-spider.log" : $"/logs/{logfile}.log";
Environment.SetEnvironmentVariable("LOGFILE", logfile);
if (Log.Logger == null)
{
var configure = new LoggerConfiguration()
#if DEBUG
.MinimumLevel.Verbose()
#else
.MinimumLevel.Information()
#endif
.MinimumLevel.Override("Microsoft", LogEventLevel.Warning)
.Enrich.FromLogContext()
.WriteTo.Console().WriteTo
.RollingFile(logfile);
Log.Logger = configure.CreateLogger();
}
Framework.SetMultiThread();
var configurationBuilder = new ConfigurationBuilder();
configurationBuilder.SetBasePath(AppDomain.CurrentDomain.BaseDirectory);
configurationBuilder.AddCommandLine(args, Framework.SwitchMappings);
configurationBuilder.AddEnvironmentVariables();
var configuration = configurationBuilder.Build();
var id = configuration["DOTNET_SPIDER_ID"] ?? Guid.NewGuid().ToString("N");
var name = configuration["DOTNET_SPIDER_NAME"] ?? id;
var arguments = Environment.GetCommandLineArgs();
var builder = new SpiderHostBuilder();
builder.ConfigureLogging(b =>
{
#if DEBUG
b.SetMinimumLevel(LogLevel.Debug);
#else
b.SetMinimumLevel(LogLevel.Information);
#endif
b.AddSerilog();
});
var config = configuration["DOTNET_SPIDER_CONFIG"];
builder.ConfigureAppConfiguration(x =>
{
if (!string.IsNullOrWhiteSpace(config) && File.Exists(config))
{
// 添加 JSON 配置文件
x.AddJsonFile(config);
}
else
{
if (File.Exists("appsettings.json"))
{
x.AddJsonFile("appsettings.json");
}
}
x.AddCommandLine(Environment.GetCommandLineArgs(), Framework.SwitchMappings);
x.AddEnvironmentVariables();
});
builder.ConfigureServices(services =>
{
services.AddThroughMessageQueue();
services.AddLocalDownloadCenter();
services.AddDownloaderAgent(x =>
{
x.UseFileLocker();
x.UseDefaultAdslRedialer();
x.UseDefaultInternetDetector();
});
services.AddStatisticsCenter(x =>
{
// 添加内存统计服务
x.UseMemory();
});
});
var spiderType = typeof(TSpider);
builder.Register(spiderType);
var provider = builder.Build();
var instance = provider.Create(spiderType);
if (instance != null)
{
instance.Name = name;
instance.Id = id;
instance.RunAsync(arguments).ConfigureAwait(true).GetAwaiter().GetResult();
}
else
{
Log.Logger.Error("Create spider object failed", 0, ConsoleColor.DarkYellow);
}
}
protected abstract void ConfigureService(IConfiguration configuration, SpiderHostBuilder builder);
/// <summary>
/// 运行
/// </summary>
/// <param name="args">运行参数</param>
public void Execute(params string[] args)
{
try
{
var logfile = Environment.GetEnvironmentVariable("DOTNET_SPIDER_ID");
logfile = string.IsNullOrWhiteSpace(logfile) ? "dotnet-spider.log" : $"/logs/{logfile}.log";
Environment.SetEnvironmentVariable("LOGFILE", logfile);
ConfigureSerialLog(logfile);
Framework.SetMultiThread();
var configurationBuilder = new ConfigurationBuilder();
configurationBuilder.SetBasePath(AppDomain.CurrentDomain.BaseDirectory);
configurationBuilder.AddCommandLine(args, Framework.SwitchMappings);
configurationBuilder.AddEnvironmentVariables();
var configuration = configurationBuilder.Build();
string spiderTypeName = configuration["DOTNET_SPIDER_TYPE"];
if (string.IsNullOrWhiteSpace(spiderTypeName))
{
Log.Logger.Error("There is no specified spider type");
return;
}
var name = configuration["DOTNET_SPIDER_NAME"];
var id = configuration["DOTNET_SPIDER_ID"] ?? Guid.NewGuid().ToString("N");
var arguments = configuration["DOTNET_SPIDER_ARGS"]?.Split(' ');
var spiderTypes = DetectSpiders();
if (spiderTypes == null || spiderTypes.Count == 0)
{
return;
}
var spiderType = spiderTypes.FirstOrDefault(x =>
x.UnderlyingSystemType.ToString().ToLower() == spiderTypeName.ToLower());
if (spiderType == null)
{
Log.Logger.Error($"Spider {spiderTypeName} not found", 0, ConsoleColor.DarkYellow);
return;
}
var builder = new SpiderHostBuilder();
ConfigureService(configuration, builder);
builder.Register(spiderType);
var provider = builder.Build();
var instance = provider.Create(spiderType);
if (instance != null)
{
instance.Name = name;
instance.Id = id;
ConfigureSpider(instance);
instance.RunAsync(arguments).ConfigureAwait(true).GetAwaiter().GetResult();
}
else
{
Log.Logger.Error("Create spider object failed", 0, ConsoleColor.DarkYellow);
}
}
catch (Exception e)
{
Log.Logger.Error($"Execute spider failed: {e}");
}
}
