public static void Initialize(CommonBootStrapper bootStrapper)
{
_bootStrapper = bootStrapper;
}
public static void Initialize(CommonBootStrapper bootStrapper)
{
_bootStrapper = bootStrapper;
}
static void Main(string[] args)
{
// IOC的注入
BootStrapperManager.Initialize(new NinjectBootstrapper());
WebBrowerManager.Instance.Setup(new cEXWB());
WebBrowerManager.Instance.FilterRequest = true;
WebBrowerManager.Instance.FilterAction.Add(
".css",
(string key, string source) =>
{
if (source.EndsWith(key))
{
return(true);
}
return(false);
});
WebBrowerManager.Instance.FilterAction.Add(
".aspx",
(string key, string source) =>
{
if (source.Contains(key))
{
return(true);
}
return(false);
});
WebBrowerManager.Instance.FilterAction.Add(
".ashx",
(string key, string source) =>
{
if (source.Contains(key))
{
return(true);
}
return(false);
});
WebBrowerManager.Instance.FilterAction.Add(
"http://wiki.360buy.com",
(string key, string source) =>
{
if (source.StartsWith(key))
{
return(true);
}
return(false);
});
WebBrowerManager.Instance.FilterAction.Add(
"http://chat.360buy.com",
(string key, string source) =>
{
if (source.StartsWith(key))
{
return(true);
}
return(false);
});
while (Process.GetCurrentProcess().WorkingSet64 / 1024 / 1024 < 100)
{
Console.WriteLine(Process.GetCurrentProcess().WorkingSet64 / 1024 / 1024);
try
{
var common = CommonBootStrapper.GetInstance <ICommonSpider>();
var spiderTasks = common.GetSpiderTask(20);
if (spiderTasks == null || spiderTasks.Count == 0)
{
Process.GetCurrentProcess().Kill();
return;
}
int i = 0;
var serverToClient = CommonBootStrapper.GetInstance <ISpiderClientToManageClient>();
foreach (var spiderProductInfo in spiderTasks)
{
i++;
var document = WebBrowerManager.Instance.Brower(spiderProductInfo.Url);
serverToClient.TransferData(new SpiderResult()
{
Url = document.Url,
TaskCount = spiderTasks.Count,
Current = i,
HtmlSource = document.HtmlSource,
Title = document.Title,
Elapse = document.Elapse,
Charset = document.Encoding
});
if (i == 1)
{
serverToClient.ReportIEVersion(WebBrowerManager.Instance.IEVersion);
}
}
}
catch (Exception exception)
{
Process.GetCurrentProcess().Kill();
}
}
Process.GetCurrentProcess().Kill();
}
public BaseController()
{
authenticationService = CommonBootStrapper.GetInstance <IAuthenticationService>();
}