/// <summary>
/// 开启套接字监听
/// 向主节点发送登记，登记成功，并返回可用的端口，然后开启指定端口的子节点server
/// </summary>
public static void Start()
{
try
{
if (false == GlobalContext.IsConfigClusteringMode)//是否开启集群模式
{
return;
}
//一旦主控节点开启并正确返回结果
if (!MasterRemoteServer.IsMasterStarted())
{
return;
}
//进入监控进程broker
AppBroker.MonitorIsMasterCenterAlive();
int port = 0;
var slaveIdentity = Guid.NewGuid().ToString().ToLower();
//开启监听前 ,发送注册当前从节点到主节点，如果可以登记注册成功，那么服务端分配端口
port = MasterRemoteServer.RegisterSlaveToMaster(slaveIdentity);
int counter = 1;
//注册失败后 再次尝试3次
while (port <= 0 && counter <= 3)
{
port = MasterRemoteServer.RegisterSlaveToMaster(slaveIdentity);
counter += 1;
RunningLocker.CreateNewLock().CancelAfter(1000);//延迟并发，防止端口资源抢占
}
if (port <= 0)
{
Console.WriteLine("未成功注册从节点端口!");
return;//一旦服务端返回无效端口 那么禁止从节点启动监听
}
listener = new NTCPMessage.Server.NTcpListener(new IPEndPoint(IPAddress.Any, port));
listener.DataReceived += new EventHandler <ReceiveEventArgs>(ReceiveEventHandler);
listener.ErrorReceived += new EventHandler <ErrorEventArgs>(ErrorEventHandler);
listener.RemoteDisconnected += new EventHandler <DisconnectEventArgs>(DisconnectEventHandler);
GlobalContext.IsInSlaveMode = true;//标识正在从节点下工作
//开启从节点的监听
listener.Listen();
}
catch (Exception ex)
{
Logger.Error(ex);
}
System.Threading.Thread.Sleep(System.Threading.Timeout.Infinite);
}
/// <summary>
/// 构造函数
/// </summary>
/// <param name="SellerId"></param>
/// <param name="ItemId"></param>
public YouhuiquanExistsTaskBuffer(long SellerId, long ItemId)
{
this.cancelTokenSource = new CancellationTokenSource();
this.TaskBufferQueue = new System.Collections.Concurrent.ConcurrentQueue <FuncForQueryQuanExists>();
this.OnQueryQuanComplete = this.OnCompleteTaskHandler;
//this.TaskAssertFunc = this.OnCompleteTaskHandler;
this.ResultModel = new YouhuiquanExistsModel();
this.ResultModel.SellerId = SellerId;
this.ResultModel.ItemId = ItemId;
//构建任务标识 空委托，依赖取消任务标识 判定任务的完成
this.QueryTaskEndPoint = new Task(() =>
{
RunningLocker.CreateNewLock().Pause();
}, this.cancelTokenSource.Token);
}
public static int Init(string[] args)
{
//1 设定当前程序运行的主上下文
GlobalContext.SyncContext = SynchronizationContext.Current;
//1-1 集群节点的判定 一旦进程启动参数有此标识，那么表示是子节点进程需要启动
//子节点仅仅用来做负载均衡的分流，不承载cef 运行时
//"type=slavemode mainProcessId={0}";
if (null != args &&
string.Concat(args).Contains("slavemode"))
{
GlobalContext.IsInSlaveMode = true;
//从启动参数 获取主进程的id
var paramProcess = args.FirstOrDefault(x => x.Contains("mainProcessId"));
if (null != paramProcess)
{
GlobalContext.MainProcessId = int.Parse(paramProcess.Split('=')[1]);
SlaveRemoteServer.Start();
}
return(0);//子节点的进程启动完毕后，返回
}
//2 初始化CEF运行时
#region 初始化CEF运行时
try
{
//加载CEF 运行时 lib cef
CefRuntime.Load();
var mainArgs = new CefMainArgs(args);
var app = new HeadLessWebBrowerApp();
//执行CEF启动进程，如果是二级进程 ，那么执行后返回exitCode，启动进程是 -1
/*// CefExecuteProcess returns -1 for the host process
* CefExecuteProcess()方法来检测是否要启动其它的子进程。此处的CefExecuteProcess是在libcef_dll_wrapper.cc中的，它内部又调用了cef_execute_process方法（libcef_dll.cc），cef_execute_process又调用了libcef/browser/context.cc文件内实现的CefExecuteProcess方法
* 它分析了命令行参数，提取”type”参数，如果为空，说明是Browser进程，返回-1，这样一路回溯到wWinMain方法里，然后开始创建Browser进程相关的内容。
* 如果”type”参数不为空，做一些判断，最后调用了content::ContentMain方法，直到这个方法结束，子进程随之结束。
* */
var exitCode = CefRuntime.ExecuteProcess(mainArgs, app, IntPtr.Zero);
if (exitCode != -1)
{
return(exitCode);
}
//一旦为 -1 ,那么表示为Host 进程，对cef 运行时进行初始化操作---
// CEF的配置参数，有很多参数，我们这里挑几个解释一下：
//SingleProcess = false：此处目的是使用多进程。
//注意： 强烈不建议使用单进程，单进程不稳定，而且Chromium内核不支持
//MultiThreadedMessageLoop = true：此处的目的是让浏览器的消息循环在一个单独的线程中执行
//注意： 强烈建议设置成true,要不然你得在你的程序中自己处理消息循环；自己调用CefDoMessageLoopWork()
//Locale = "zh-CN"：webkit用到的语言资源，如果不设置，默认将为en - US
//注意： 可执行文件所在的目录一定要有locals目录，而且这个目录下要有相应的资源文件
var settings = new CefSettings
{
// BrowserSubprocessPath = @"D:\fddima\Projects\Xilium\Xilium.CefGlue\CefGlue.Demo\bin\Release\Xilium.CefGlue.Demo.exe",
SingleProcess = false, //开启多进程模型，tab 进程独立模式
MultiThreadedMessageLoop = true, //开启多线程任务
WindowlessRenderingEnabled = true, //开启无头模式
//Locale = "en-US",
LogSeverity = CefLogSeverity.Disable,
//LogFile = "CefGlue.log",
PersistSessionCookies = true,
UserAgent = GlobalContext.ChromeUserAgent,
UserDataPath = System.IO.Path.Combine(Environment.CurrentDirectory, "UserData"),
CachePath = System.IO.Path.Combine(Environment.CurrentDirectory, "LocalCache")
};
// DevTools doesn't seem to be working when this is enabled
// http://magpcss.org/ceforum/viewtopic.php?f=6&t=14095
// Disable GPU in WPF and Offscreen examples until #1634 has been resolved
//settings.CefCommandLineArgs.Add("disable-gpu", "1");
//初始化 CEF进程参数设置
CefRuntime.Initialize(mainArgs, settings, app, IntPtr.Zero);
}
catch (Exception ex)
{
Logger.Error(ex);
return(3);
}
#endregion
//3 开启总控TCP端口，用来接收站点的请求--开启后 会阻塞进程 防止结束
// 总控端口 负责 1 收集请求 响应请求 2 收集分布的采集客户端 登记注册可用的端，用来做CDN 任务分发，做负载均衡
MasterRemoteServer.Start();
#region 开启集群模式
if (GlobalContext.IsConfigClusteringMode)
{
var clusterNodeCount = ConfigHelper.GetConfigInt("ClusterNodeCount");
if (clusterNodeCount <= 0)
{
clusterNodeCount = 1;
}
try
{
//清理残留进程
AppBroker.ClearGarbageProcess();
var mainProcess = Process.GetCurrentProcess();
string appName = Assembly.GetExecutingAssembly().GetName().Name;
//开启子节点进程
for (int i = 0; i < clusterNodeCount; i++)
{
Process p = new Process();
p.StartInfo.FileName = System.IO.Path.Combine(AppDomain.CurrentDomain.BaseDirectory, string.Format("{0}.exe", appName));
p.StartInfo.Arguments = string.Format(GlobalContext.SlaveModelStartAgrs, mainProcess.Id);//集群节点的启动参数标识
p.StartInfo.UseShellExecute = true;
p.StartInfo.WindowStyle = ProcessWindowStyle.Hidden;
p.Start();
//延迟启动下一个从节点进程，防止并发分配端口资源导致失败
RunningLocker.CreateNewLock().CancelAfter(1000);
}
}
catch (Exception ex)
{
Logger.Error(ex);
}
}
#endregion
//4 定时清理器
#region 定时清理控制台
//ConsoleClean.Start();
#endregion
//5 阿里妈妈等平台登录
#region 阿里妈妈等平台网页初始化操作
//----------注意：由于开启了多进程模型，不同的网址在不同的tab,每个tab 在其独立的进程中-------
//每次打开一个程序进程，让进程开启一个端口server,向总控端口，发送注册登记，用来接受请求转发，做负载均衡---
//初始化平台网页进程
//1 通过反射 获取所有的webpage service --正式版
var ass = Assembly.GetExecutingAssembly();
var typeFinder = new AppDomainTypeFinder();
var targetType = typeof(BaseWebPageService);
var webPageServiceTypes = typeFinder.FindClassesOfType(targetType, new Assembly[] { ass }, true);
if (webPageServiceTypes.IsNotEmpty())
{
foreach (Type itemPageService in webPageServiceTypes)
{
#region 没用的服务 先不启用
if (typeof(VipWebPageService).Equals(itemPageService))
{
continue;
}
if (typeof(MogujieWebPageService).Equals(itemPageService))
{
continue;
}
if (typeof(YhdWebPageService).Equals(itemPageService))
{
continue;
}
#endregion
try
{
BaseWebPageService servieInstance = Activator.CreateInstance(itemPageService) as BaseWebPageService;
//静态属性访问一次 即可触发打开页面
var loader = servieInstance.RequestLoader;
}
catch (Exception ex)
{
Logger.Error(ex);
}
}
}
// --------------测试环境begin 不建议打开多个tabpage，影响测试加载-------------
//var tmallService = new TmallWebPageService();
//var loader = tmallService.RequestLoader;
//var loader_taobao = new TaobaoWebPageService().RequestLoader;
//var loader_jd = new JingdongWebPageService().RequestLoader;
//var loader_dangdang = new DangdangWebPageService().RequestLoader;
//--------------测试环境end-------使用一个tabpage ,即可测试是否正确加载----------
#endregion
//6 监视残留的render进程
AppBroker.MonitorClearRenderProcessByLifeTime();
return(0);
}
private static void Main(string[] args)
{
//破解淘宝客加密的链接
//string url = "http://s.click.taobao.com/t?spm=1002.8113010.1999451588.1.197829d2xOjGWY&e=m%3D2%26s%3DmP8QGUZCl18cQipKwQzePOeEDrYVVa64LKpWJ%2Bin0XK3bLqV5UHdqU7FFcTKJEXpBuky%2F0Sep%2BFpvEi8xmC0PQfgGrPFD%2FD7ItsJf7xhZUukOrdzMLy3g0C9MWo3ZAy5ZtvIAOb0yL8buZkKjgqa4LRqys2RxTiLmiP8wiUuCvFDEV8PXh1a5UciGQ2l2vvBJoe7ipwP0MtRLBgaW5udaw%3D%3D";
//string content = new Http.HttpServerProxy().GetRequestTransfer(url,null);
//var rl = TaobaoWebPageService.GetTaobaoUnionOfficalUrl(url);
// return;
try
{
//初始化CEF运行时 等操作
InitApp.Init(args);
}
catch (Exception ex)
{
// CefRuntime.PostTask(CefThreadId.IO, new MySomeTask());
// Clean up CEF.
CefRuntime.Shutdown();
MasterRemoteServer.Stop();
Logger.Error(new Exception("未能正确启动CEF爬行蜘蛛！异常信息如下："));
Logger.Error(ex);
return;
}
//var locker1 = RunningLocker.CreateNewLock();
//locker1.CancelAfter(20000);
//BaseWebPageService etaoWeb = new TaobaoWebPageService();
//var paras = new NTCPMessage.EntityPackage.Arguments.TaobaoFetchWebPageArgument { KeyWord = "洗面奶男" };
//var con = etaoWeb.QuerySearchContent(paras);
//System.Diagnostics.Debug.WriteLine(con.Result);
//etaoWeb = new JingdongWebPageService();
// con = etaoWeb.QuerySearchContent(paras);
//var headlessForm = new HeadLessMainForm();
//headlessForm.NavigateToUrl("https://pub.alimama.com/myunion.htm?spm=a219t.7900221/1.1998910419.dbb742793.21214865YeCJuR#!/promo/self/items");
Console.WriteLine("ShoppingWebCrawler.Host is started.....");
if (null != args &&
string.Concat(args).Contains("slavemode"))
{
Console.WriteLine("SlaveRemoteServer is started.....");
}
//注册窗口关闭的时候 退出子进程
// we have to keep the handler routine alive during the execution of the program,
// because the garbage collector will destroy it after any CTRL event
GC.KeepAlive(cancelHandler);
SetConsoleCtrlHandler(cancelHandler, true);
var locker = RunningLocker.CreateNewLock();
locker.Pause();
//6 主进程退出的事件
}
/// <summary>
/// 自身健康监测
/// </summary>
/// <param name="callbackHandlerOnFailed">失败的时候 执行的委托回调</param>
public void BeginSelfHelthCheck(Action <string> callbackHandlerOnFailed)
{
if (isRunningHelthCheck == true)
{
return;
}
//1 向此节点对应的端口 发送ping
//2 失败3次 定性为错误无效节点
Task.Factory.StartNew(() =>
{
while (true)
{
RunningLocker.CreateNewLock().CancelAfter(1000);
try
{
bool isBeUsed = SocketHelper.IsUsedIPEndPoint(this.Port);
if (isBeUsed == false)
{
if (counterStatusCheck >= 10)
{
this.IsActiveNode = false;
if (null != callbackHandlerOnFailed)
{
callbackHandlerOnFailed(this.Identity);
}
break;
}
else
{
counterStatusCheck += 1;
continue;//端口暂未开启
}
}
using (var conn = new SoapTcpConnection(this.IpAddress, this.Port, 4))
{
if (conn.State != ConnectionState.Open)
{
conn.Open();
}
var result = conn.Ping();//发送Ping 命令，是否可以ping 通子节点端口
if (result != true)
{
//一旦超过3此ping 不通，表示这是一个坏节点，中集群中心移除此节点--进行回调通知
if (this.failCounter >= 2)
{
this.IsActiveNode = false;
if (null != callbackHandlerOnFailed)
{
callbackHandlerOnFailed(this.Identity);
}
break;
}
this.failCounter += 1;
}
}
}
catch (Exception ex)
{
this.IsActiveNode = false;
Common.Logging.Logger.Error(ex);
break;
}
}
});
}
//private static int count = 0;
/// <summary>
/// 开启N 个线程，将从redis 中抓取数据进行消费
/// </summary>
private static void ProcessMessageDataQueue()
{
//var tm = new System.Timers.Timer();
//tm.Interval = 1000;
//tm.Elapsed += (s, e) => {
// System.Diagnostics.Debug.WriteLine(count);
//};
//tm.Start();
for (int i = 0; i < ExcuteThreadCount; i++)
{
var th = new Thread(new ThreadStart(() =>
{
IMessageProcessor QianLangQuanProcessor = new QianLangPlanQuanProcessor();
_LstProcessors.Enqueue(QianLangQuanProcessor);
while (true)
{
try
{
if (_token.IsCancellationRequested == true)
{
break;//终止信号
}
////如果集合都是空 那么等待休眠
if (QianLangPlanQuanProcessor.MessageDataQueueOfQianLangQuan.IsEmpty == true)
{
RunningLocker.CreateNewLock().CancelAfter(1000 * 2);//休眠2秒
}
//count++;
//并行执行
Parallel.Invoke(
() =>
{
//处理消息
if (QianLangPlanQuanProcessor.MessageDataQueueOfQianLangQuan.IsEmpty)
{
return;
}
var pair = QianLangPlanQuanProcessor.PopOneQianLangQuanMessage();
if (default(KeyValuePair <string, IMessage>).Equals(pair))
{
return;
}
string key = pair.Key;
IMessage itemFee = pair.Value;
QianLangQuanProcessor.Process(key, itemFee);
}
);
}
catch (Exception ex)
{
Logger.Error(ex);
}
}
}));
th.IsBackground = true;
th.Priority = ThreadPriority.Normal;
//压入处理线程队列
_TheadQueuePool_Processor.Enqueue(th);
th.Start();
}
}
/// <summary>
/// 从redis 中不间断的批量读取消息
/// </summary>
private static void FetchDataFromRedis()
{
while (true)
{
RedisCacheManager redisClient = null;
try
{
if (_token.IsCancellationRequested == true)
{
break;//终止信号
}
//目前仅处理千浪计划消息
string matchPattern = string.Concat(BaseMessageModel.CachePrefix, "*");
//Logging.Logger.Info("组件匹配的键模式："+matchPattern);
redisClient = new RedisCacheManager();
var scanResult = redisClient.Scan(matchPattern, 10, _redis_scan_cursor);
#region 限流策略---针对不同的积压阶段实施限流措施，防止拖死进程和redis
//游标置为0 的时候 标识已经回到起始的位置
if (null == scanResult || _redis_scan_cursor == 0)
{
_sleepTimeSpan = 1000;
RunningLocker.CreateNewLock().CancelAfter(_sleepTimeSpan);
if (null == scanResult)
{
continue;//如果null 结果 那么不用继续下面的逻辑
}
}
//积压2000
if (QianLangPlanQuanProcessor.MessageDataQueueOfQianLangQuan.Count > 2000)
{
_sleepTimeSpan = 2000;
RunningLocker.CreateNewLock().CancelAfter(_sleepTimeSpan);
}
//积压5000
if (QianLangPlanQuanProcessor.MessageDataQueueOfQianLangQuan.Count > 5000)
{
_sleepTimeSpan = 3000;
RunningLocker.CreateNewLock().CancelAfter(_sleepTimeSpan);
}
if (QianLangPlanQuanProcessor.MessageDataQueueOfQianLangQuan.Count > 100000)
{
Logger.Error(new Exception("消费队列过于缓慢！内存积压严重！"));
break;
}
#endregion
//回写游标
_redis_scan_cursor = scanResult.Cursor;
var allKeys = scanResult.Results;
if (allKeys.IsEmpty())
{
RunningLocker.CreateNewLock().CancelAfter(1000);
continue;
}
if (null != allKeys && allKeys.IsNotEmpty())
{
//var redisClient = new RedisCacheManager().RedisClient;
foreach (var key in allKeys)
{
//Logging.Logger.Info("恭喜大人，key is："+key);
try
{
var isInHistory = CheckIsInProcessHistory(key);
if (isInHistory == true)
{
continue;
}
else
{
LocalProcessdHistoryHashTable.Add(key);
}
//处理管线
if (key.Contains(ProcessPipeLine.QianLangPlanQuan.ToString()))
{
//将当前扫描的键 获取对象结果，添加到集合中
QianLangQuanMessage item = redisClient.Get <QianLangQuanMessage>(key);
if (null != item)
{
if (!QianLangPlanQuanProcessor.MessageDataQueueOfQianLangQuan.ContainsKey(key))
{
QianLangPlanQuanProcessor.MessageDataQueueOfQianLangQuan.TryAdd(key, item);
}
}
}
else
{
redisClient.Remove(key);
continue;//无效的消息 直接移除掉
}
}
catch (Exception ex)
{
Logger.Error(ex);
}
}
}
}
catch (Exception ex)
{
Logger.Error(ex);
}
finally
{
if (null != redisClient)
{
redisClient.Dispose();
}
}
}
}