protected override void OnStartup()
{
base.CreateLiveNode("/live_nodes/crawler/" + BaseUrl, null);
//create crawler config in zookeeper
var stat = zooKeeper.existsAsync("/config/crawler/" + BaseUrl, false).Result;
if (stat == null)
{
var d = new CrawlerConfig()
{
Name = BaseUrl,
baseUrl = BaseUrl,
Proxy = ProxyUrl,
Ips = new string[0],
UseCookie = true
};
zooKeeper.createAsync("/config/crawler/" + BaseUrl, Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(d)), Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
}
}
protected override void OnStartup()
{
base.CreateLiveNode("/live_nodes/crawler/" + BaseUrl, null);
//create crawler config in zookeeper
var stat = zooKeeper.Exists("/config/crawler/" + BaseUrl, false);
if (stat == null)
{
var d = new CrawlerConfig()
{
Name = BaseUrl,
baseUrl = BaseUrl,
Proxy = ProxyUrl,
Ips = new string[0],
UseCookie = true
};
zooKeeper.Create("/config/crawler/" + BaseUrl, JsonConvert.SerializeObject(d).GetBytes(), Ids.OPEN_ACL_UNSAFE, CreateMode.Persistent);
}
}
