/// <summary>
/// Constructor for Halloween effect
/// </summary>
/// <param name="chessboard">Chessboard background</param>
/// <param name="sound">Sound system</param>
/// <param name="NumberOfSpiders">How many spiders to show on screen</param>
public Halloween(ref Chess chessboard, ref Sound sound, int NumberOfSpiders)
{
chess = chessboard;
snd = sound;
textureSpider = Util.LoadTexture(Util.CurrentExecutionPath + "/gfx/spider.bmp", TextureMinFilter.Linear, TextureMagFilter.Linear, TextureWrapMode.Clamp, TextureWrapMode.Clamp, Color.Black);
textureGhost = Util.LoadTexture(Util.CurrentExecutionPath + "/gfx/halloween.bmp", TextureMinFilter.Linear, TextureMagFilter.Linear, TextureWrapMode.Clamp, TextureWrapMode.Clamp, Color.FromArgb(255,0,255));
snd.CreateSound(Sound.FileType.Ogg, Util.CurrentExecutionPath + "/Samples/scary.ogg", "Scary");
MaxSpiders = NumberOfSpiders;
Spiders = new Spider[MaxSpiders];
Ghost = new Vector3[4];
Size = new SizeF(1.0f, 1.0f);
float sZ = 0.4f;
for (int i = 0; i < MaxSpiders; i++)
{
Spiders[i] = new Spider(0.1f, 0.1f, sZ, 1.0f, textureSpider);
sZ -= sZ / (NumberOfSpiders + 2000);
}
X = Util.Rnd.Next(-3, 3) / 10.0f;
Y = Util.Rnd.Next(-3, 3) / 10.0f;
Z = sZ - 0.001f;
Ghost[0] = new Vector3(X, Y, Z); // red
Ghost[1] = new Vector3(X, Y + Size.Height, Z); // blue
Ghost[2] = new Vector3(X + Size.Width, Y + Size.Height, Z); // green
Ghost[3] = new Vector3(X + Size.Width, Y, Z); // yellow
LastPlayedDate = string.Empty;
}
public void Start_Started_ReturnsTrue()
{
var httpClientHelper = A.Fake<IHttpClientHelper>();
var spider = new Spider(A.Fake<ISettings>(), A.Fake<ILogger>(), httpClientHelper);
A.CallTo(() => httpClientHelper.MakeHttpRequest(A<string>.Ignored, HttpResponseType.json)).Returns("{\"Result\":\"OK\"}");
bool hasStarted = spider.Start("");
Assert.AreEqual(true, hasStarted);
}
public void HasCompleted_NotComplete_ReturnsFalse()
{
var httpClientHelper = A.Fake<IHttpClientHelper>();
var spider = new Spider(A.Fake<ISettings>(), A.Fake<ILogger>(), httpClientHelper);
A.CallTo(() => httpClientHelper.MakeHttpRequest(A<string>.Ignored, HttpResponseType.json)).Returns("{\"status\":\"23\"}");
bool hasCompleted = spider.HasCompleted();
Assert.AreEqual(false, hasCompleted);
}
public static Zap Create()
{
var settings = new Settings();
var logger = new CommandLineLogger();
var httpClientHelper = new HttpClientHelper(settings);
var scanner = new Scanner(settings, logger, httpClientHelper);
var reporting = new Reporting(settings,logger,httpClientHelper);
var spider = new Spider(settings, logger, httpClientHelper);
return new Zap( settings, logger, scanner, reporting, spider);
}
public void Start_NullResponse_ReturnsFalse()
{
var httpClientHelper = A.Fake<IHttpClientHelper>();
var spider = new Spider(A.Fake<ISettings>(), A.Fake<ILogger>(), httpClientHelper);
A.CallTo(() => httpClientHelper.MakeHttpRequest(A<string>.Ignored, A<HttpResponseType>.Ignored)).Returns(null);
bool hasStarted = spider.Start("");
Assert.AreEqual(false, hasStarted);
}
private static CrawlResults RunCrawler(Season season)
{
Log.InfoFormat("Crawler started at {0} for season {1}", DateTime.Now.ToShortTimeString(), season.Name);
Spider crawler = new Spider();
CrawlResults results = crawler.Crawl(season);
Log.InfoFormat("\n{0}", DumpResults(results));
Log.InfoFormat("Crawler finished at {0}.", DateTime.Now.ToShortTimeString());
return results;
}
public PesquisaWeb(string username)
{
log = new LoginDAL();
histBLL = new HistoricoBLL();
int[] defs = log.getDefinicoes(username);
this.nLinksSeed = Convert.ToUInt32(defs[0]);
this.profundidade = defs[1];
this.modoPesquisa = defs[2];
spider = new Spider(profundidade);
}
static void RunSpider()
{
var bloomFilter = new MemoryBloomFilter<string>(1000 * 10, 1000 * 10 * 20);
_spider = new Spider(new SequenceScheduler(bloomFilter));
var downloaders = new List<IDownloader> { new HttpClientDownloader(4) };
_spider.RegisterDownloader(downloaders);
_spider.RegisterPageAnalyzer<UsashopcnPageAnalyzer>(UsashopcnPageAnalyzer.SiteId);
_spider.RegisterResultPipeModule(new ConsoleModule(0, 20, 400, 500, true, true));
_spider.Start(TopicType.StaticHtml, SiteIndex.Usashopcn, "http://www.usashopcn.com/");
var statusTimer = new Timer(spider => { Console.WriteLine(((Spider) spider).RunStatusInfo()); }, _spider, 0, 2000);
}
private static void RecheckSeason(FileStore dataStore, Season season)
{
CrawlResults existing = GetCrawlResultsForSeason(dataStore, season);
if (existing == null)
{
Log.WarnFormat("Season {0} has not been crawled yet.", season.Name);
return;
}
Spider spider = new Spider();
CrawlResults recheckResults = spider.Recheck(existing);
Log.InfoFormat("\n{0}", DumpResults(recheckResults));
Log.InfoFormat("Crawler finished at {0}.", DateTime.Now.ToShortTimeString());
SaveCrawlerResults(recheckResults, dataStore);
}
public void RunSpider()
{
var bloomFilter = new MemoryBloomFilter<string>(1000 * 10, 1000 * 10 * 20);
_spider = new Spider(new SequenceScheduler(bloomFilter));
//var downloaders = new List<IDownloader> { new FakeDownloader(4) };
var downloaders = new List<IDownloader> { new HttpClientDownloader(4) };
_spider.RegisterDownloader(downloaders);
_spider.RegisterPageAnalyzer<UsashopcnPageAnalyzer>(UsashopcnPageAnalyzer.SiteId);
_spider.RegisterResultPipeModule(new ConsoleModule(500, 0, 400, 500, true, true));
_spider.Start(TopicType.StaticHtml, SiteIndex.Usashopcn, GetUrls());
var statusTimer = new Timer(spider =>
{
var status = ((Spider) spider).RunStatusInfo();
Console.WriteLine(String.Format("QueueCount:{0}, TaskCount={1}, ConsumeTotal:{2}, ResultTotal:{3}", status.QueueCount, status.TaskCount, status.ConsumeTotal, status.ResultTotal));
}, _spider, 2000, 2000);
}
/// <summary>
/// 启动
/// </summary>
/// <param name="request">抓取请求</param>
/// <param name="parser">页面分析器</param>
/// <param name="storage">数据存储器,,默认显示在控制台</param>
public void Start(List <Request> request, List <IDataFlow> parser, IDataFlow storage = null)
{
Spider _spider = _provider.Create <Spider>();
if (null == storage)
{
storage = new ConsoleStorage();
}
_spider.NewGuidId(); // 设置任务标识
_spider.Name = "测试采集"; // 设置任务名称
_spider.Speed = 10; // 设置采集速度, 表示每秒下载多少个请求, 大于 1 时越大速度越快, 小于 1 时越小越慢, 不能为0.
_spider.Depth = 3; // 设置采集深度
if (parser != null)
{
foreach (IDataFlow item in parser)
{
_spider.AddDataFlow(item);
}
}
_spider.AddDataFlow(storage);
_spider.AddRequests(request.ToArray()); // 设置链接
_spider.RunAsync(); // 启动
}
public static void Run()
{
// 定义要采集的 Site 对象, 可以设置 Header、Cookie、代理等
var site = new Site {
EncodingName = "UTF-8"
};
for (int i = 1; i < 5; ++i)
{
// 添加初始采集链接
site.AddStartUrl("http://www.cnblogs.com");
}
// 使用内存Scheduler、自定义PageProcessor、自定义Pipeline创建爬虫
Spider spider = Spider.Create(site,
new QueueDuplicateRemovedScheduler(),
new BlogSumaryProcessor(),
new NewsProcessor()).
AddPipeline(new MyPipeline());
// 启动爬虫
spider.Run();
}
public override void Build(Spider spider, dynamic obj)
{
Dictionary <string, object> data = new Dictionary <string, object>();
if (Extras != null)
{
foreach (var extra in Extras)
{
data.Add(extra.Key, extra.Value);
}
}
for (int i = From; i <= To; i += Interval)
{
spider.Scheduler.Push(new Request(string.Format(FormateString, i), 1, data)
{
PostBody = PostBody,
Origin = Origin,
Method = Method,
Referer = Referer
});
}
}
public static void Main(string[] args)
{
#if NET_CORE
Encoding.RegisterProvider(CodePagesEncodingProvider.Instance);
#endif
IocManager.AddSingleton <IMonitor, NLogMonitor>();
Spider spider = Spider.Create(new Site {
EncodingName = "UTF-8", MinSleepTime = 1000
}, new SpiderTest.TestPageProcessor()).AddPipeline(new SpiderTest.TestPipeline()).SetThreadNum(1);
spider.SetDownloader(new TestDownloader());
for (int i = 0; i < 10; i++)
{
spider.AddStartUrl("http://www.baidu.com/" + i);
}
spider.Run();
Thread.Sleep(5000);
spider.Stop();
Thread.Sleep(5000);
spider.RunAsync();
Thread.Sleep(5000);
}
public static void CrawlerPagesWithoutTraverse()
{
var site = new Site {
EncodingName = "UTF-8", RemoveOutboundLinks = true
};
for (int i = 1; i < 5; ++i)
{
site.AddStartUrl("http://" + $"www.youku.com/v_olist/c_97_g__a__sg__mt__lg__q__s_1_r_0_u_0_pt_0_av_0_ag_0_sg__pr__h__d_1_p_{i}.html");
}
Spider spider = Spider.Create(site,
"YOUKU_" + DateTime.Now.ToString("yyyyMMddhhmmss"),
new QueueDuplicateRemovedScheduler(),
new SimplePageProcessor())
.AddPipeline(new FilePipeline())
.SetThreadNum(2);
spider.EmptySleepTime = 3000;
// 启动爬虫
spider.Run();
}
public void RunAsyncAndStopThenExit()
{
if (Environment.GetEnvironmentVariable("TRAVIS") == "1")
{
return;
}
Spider spider = Spider.Create(new Site {
EncodingName = "UTF-8", SleepTime = 1000
}, new TestPageProcessor()).AddPipeline(new TestPipeline());
spider.ThreadNum = 1;
for (int i = 0; i < 10000; i++)
{
spider.AddStartUrl("http://www.baidu.com/" + i);
}
spider.RunAsync();
Thread.Sleep(5000);
spider.Pause(() =>
{
spider.Exit();
});
Thread.Sleep(5000);
}
static void Main(string[] args)
{
var sammie = new Cat {
Name = "Sammie", NumberOfLives = 4, Gender = GenderEnum.Male, NumberOfLegs = 4
};
var max = new Cat {
Name = "Max", NumberOfLives = 8, Gender = GenderEnum.Male, NumberOfLegs = 4
};
var jessie = new Dog {
Name = "Jessie", Gender = GenderEnum.Female, NumberOfLegs = 4, NumberOfLives = 1
};
var spider = new Spider {
Name = "Peter", NumberOfLegs = 8, NumberOfLives = 1, Gender = GenderEnum.Male
};
List <IPet> pets = new List <IPet> {
sammie, max, jessie, spider
};
CheckNumberOfLivesRemaining(pets);
MakeAnimalsSpeak(pets);
Console.ReadLine();
}
public void Setting()
{
var site = new Site {
EncodingName = "UTF-8"
};
// Set start/seed url
site.AddStartUrl("http://www.cnblogs.com/");
Spider spider = Spider.Create(site,
// crawler identity
"cnblogs_" + DateTime.Now.ToString("yyyyMMddhhmmss"),
// use memoery queue scheduler
new QueueDuplicateRemovedScheduler(),
// default page processor will save whole html, and extract urls to target urls via regex
new DefaultPageProcessor(new[] { "cnblogs\\.com" }))
// save crawler result to file in the folder: \{running directory}\data\{crawler identity}\{guid}.dsd
.AddPipeline(new FilePipeline());
PrepareAdslConfig();
spider.RedialExecutor = new MutexRedialExecutor(new AdslRedialer("adsl_account.txt"));
Assert.NotNull(spider.RedialExecutor);
}
private void InitializeApiObjects()
{
//New API needs to be instantiated here
acsrf = new Acsrf(this);
ascan = new Ascan(this);
ajaxspider = new AjaxSpider(this);
authentication = new Authentication(this);
autoupdate = new Autoupdate(this);
brk = new Break(this);
context = new Context(this);
core = new Core(this);
forcedUser = new ForcedUser(this);
httpSessions = new HttpSessions(this);
parameters = new Params(this);
pscan = new Pscan(this);
reveal = new Reveal(this);
script = new Script(this);
search = new Search(this);
selenium = new Selenium(this);
sessionManagement = new SessionManagement(this);
spider = new Spider(this);
users = new Users(this);
}
private static void PerformAdapterTest()
{
Console.WriteLine("Creating a horse and a spider...");
var horse = new Horse(4, "Samantha");
var spider = new Spider(8, "Charlie");
Console.WriteLine($"The horses name is {horse.Name()} and the spider's name is {spider.Name()}.\n");
Console.WriteLine("Verifying that we can turn the horse into a spider.\n");
if (horse.CanTurnInto(spider))
{
Console.WriteLine($"{horse.Name()} can turn into a {spider.GetType().Name}!!!\n{spider.Name()} will have a new friend!\n");
}
Console.WriteLine("Creating two boxes, one with 2 flaps another with 4 flaps.\n");
var box1 = new Box(2);
var box2 = new Box(4);
var boxAdapter1 = new BoxAdapter(box1);
var boxAdapter2 = new BoxAdapter(box2);
Console.WriteLine("Verifying if we can turn the horse into box 1...");
if (!horse.CanTurnInto(boxAdapter1))
{
Console.WriteLine($"{horse.Name()} can't turn into a box 1. Sorry.\n");
}
Console.WriteLine("Verifying if we can turn the horse into box 2...");
if (horse.CanTurnInto(boxAdapter2))
{
Console.WriteLine($"{horse.Name()} can turn into box 2!!!!\nShe will be so happy as a box!!!\n");
}
}
private static Page Crawl(string url)
{
try
{
// Config encoding, header, cookie, proxy etc... 定义采集的 Site 对象, 设置 Header、Cookie、代理等
var headers = new Dictionary <string, string>();
headers.Add("Accept", "application/xml, text/xml, */*; q=0.01");
headers.Add("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8");
var site = new Site(url)
{
EncodingName = "UTF-8", Headers = headers, RemoveOutboundLinks = true, DownloadFiles = false
};
var option = new Option(LanguageEnum.English);
var webDownloader = new WebDriverDownloader(Browser.Chrome, 200, null, option);
webDownloader.FormSubmit = new FormSubmitHandler()
{
Selectors = new List <KeyValuePair <Selector, string> >
{
new KeyValuePair <Selector, string>(new Selector(SelectorType.Id, "userId"), "*****@*****.**"),
new KeyValuePair <Selector, string>(new Selector(SelectorType.Id, "passwd"), "Worm0429@")
},
SubmitSelector = new Selector(SelectorType.XPath, "//p[@class='oauth_formbtn']/a[1]"),
ChildFormActions = GetChildFormActions()
};
Spider spider = Spider.Create(site, webDownloader);
Page page = spider.Run();
return(page);
}
catch (Exception ex)
{
Logger.Info(ex);
return(null);
}
}
public void DatebaseLogAndStatus()
{
if (File.Exists(Env.GlobalAppConfigPath))
{
File.Delete(Env.GlobalAppConfigPath);
}
AppDomain.CurrentDomain.SetData("CONFIG", "");
AppDomain.CurrentDomain.SetData("DBCONFIG", "");
Env.Reload();
string id = Guid.NewGuid().ToString("N");
string taskGroup = Guid.NewGuid().ToString("N");
string userId = Guid.NewGuid().ToString("N");
using (Spider spider = Spider.Create(new Site {
EncodingName = "UTF-8", SleepTime = 1000
},
id,
new QueueDuplicateRemovedScheduler(),
new TestPageProcessor()))
{
spider.AddPipeline(new TestPipeline());
spider.ThreadNum = 1;
for (int i = 0; i < 5; i++)
{
spider.AddStartUrl("http://www.baidu.com/" + i);
}
spider.Monitor = new DbMonitor(id);
spider.Run();
}
Thread.Sleep(3000);
using (var conn = (Env.SystemConnectionStringSettings.GetDbConnection()))
{
Assert.StartsWith("Crawl complete, cost", conn.Query <Log>($"SELECT * FROM DotnetSpider.Log where Identity='{id}'").Last().message);
Assert.Equal($"1{id}", $"{conn.Query<CountResult>($"SELECT COUNT(*) as Count FROM DotnetSpider.Status where Identity='{id}'").First().Count}{id}");
Assert.Equal("Finished", conn.Query <statusObj>($"SELECT * FROM DotnetSpider.Status where Identity='{id}'").First().status);
}
}
public void RetryRequest()
{
var site = new Site {
EncodingName = "UTF-8", RemoveOutboundLinks = true
};
var scheduler = new QueueDuplicateRemovedScheduler();
site.AddStartUrl("http://www.baidu.com");
site.AddStartUrl("http://www.cnblogs.com/");
Spider spider = Spider.Create(site,
// crawler identity
"cnblogs_" + DateTime.Now.ToString("yyyyMMddhhmmss"),
// use memoery queue scheduler
scheduler,
// default page processor will save whole html, and extract urls to target urls via regex
new TestPageProcessor())
// save crawler result to file in the folder: \{running directory}\data\{crawler identity}\{guid}.dsd
.AddPipeline(new FilePipeline());
// dowload html by http client
spider.Downloader = new HttpClientDownloader();
spider.ThreadNum = 1;
// traversal deep 遍历深度
spider.Deep = 3;
spider.EmptySleepTime = 3000;
// start crawler 启动爬虫
spider.Run();
Assert.Equal(5, spider.RetriedTimes.Value);
Assert.Equal(0, scheduler.LeftRequestsCount);
Assert.Equal(1, scheduler.SuccessRequestsCount);
// 重试次数应该包含
Assert.Equal(5, scheduler.ErrorRequestsCount);
}
public override void Build(Spider spider, dynamic obj)
{
if (string.IsNullOrEmpty(QueryString))
{
throw new SpiderException("QueryString is null.");
}
var datas = PrepareDatas();
foreach (var data in datas)
{
for (int i = From; i <= To; i += Interval)
{
var arguments = PrepareArguments(data);
arguments.Add(i.ToString());
for (int j = PostFrom; j <= PostTo; j += PostInterval)
{
foreach (var formate in FormateStrings)
{
string tmpUrl = string.Format(formate, arguments.Cast <object>().ToArray());
spider.Scheduler.Push(new Request(tmpUrl, 0, data)
{
Method = Method,
Origin = Origin,
PostBody = GetPostBody(PostBody, data, j),
Referer = Referer
});
}
if (!string.IsNullOrEmpty(CookieString))
{
spider.Site.CookiesStringPart = string.Format(CookieString, arguments.Cast <object>().ToArray());
}
}
}
}
}
override public void PROCESSOR(BotCycle bc)
{
CustomBot cb = (CustomBot)bc.Bot;
if (!cb.HR.Get(Url))
{
throw new ProcessorException(ProcessorExceptionType.RESTORE_AS_NEW, "Could not get: " + Url);
}
DataSifter.Capture gc = cb.product.Parse(cb.HR.HtmlResult);
decimal stock = (decimal)Fhr.CrawlerHost.Product.StockValue.NOT_SET;
if (gc.ValueOf("Stock") != null)
{
if (!decimal.TryParse(gc.ValueOf("Stock"), out stock))
{
stock = (decimal)Fhr.CrawlerHost.Product.StockValue.CANNOT_PARSE;
}
}
Fhr.CrawlerHost.Product product = new Fhr.CrawlerHost.Product(
id: gc.ValueOf("Id"),
url: Url,
name: gc.ValueOf("Name"),
sku: gc.ValueOf("Sku"),
price: gc.ValueOf("Price"),
category_branch: gc.ValuesOf("Category"),
image_urls: Spider.GetAbsoluteUrls(gc.ValuesOf("ImageUrl"), Url, cb.HR.HtmlResult),
stock: stock,
description: gc.ValueOf("Description")
);
if (!Fhr.CrawlerHost.CrawlerApi.SaveProductAsJson(product))
{
throw new ProcessorException(ProcessorExceptionType.ERROR, "Product was not saved.");
}
}
private void downloadLink_Click(object sender, RoutedEventArgs e)
{
Hyperlink link = sender as Hyperlink;
string url = link.NavigateUri.AbsoluteUri;
Task.Run(async() =>
{
Spider s = new Spider();
var content = await s.GetResponse(url, "get");
HtmlDocument doc = new HtmlDocument();
doc.LoadHtml(content);
string downloadUrl = doc.DocumentNode.SelectNodes(@"//html/body/div[2]/div[2]/div[3]/div[2]/span[1]/a")[0].Attributes["href"].Value;
try
{
Process.Start(new ProcessStartInfo(downloadUrl));
}
catch (Exception ex) //容错处理 - 用户没有设置默认浏览器时-则使用IE浏览器打开
{
//调用IE浏览器
Process.Start("firefox.exe", downloadUrl);
}
});
}
private void RegisterControl(Spider spider)
{
if (Redis != null)
{
try
{
Redis.GetSubscriber().Subscribe($"{spider.Identity}", (c, m) =>
{
switch (m)
{
case "STOP":
{
spider.Stop();
break;
}
case "RUNASYNC":
{
spider.RunAsync();
break;
}
case "EXIT":
{
spider.Exit();
break;
}
}
});
}
catch
{
// ignored
}
}
}
private void renamebutton_Click(object sender, EventArgs e)
{
try
{
if (folderlabel.Text != "" && folderlabel.Text != "语料库文件夹路径")
{
Spider searchSpider = new Spider(folderlabel.Text);
Action longAction = new Action(searchSpider.RenameFileName);
IAsyncResult re = longAction.BeginInvoke(null, null);
while (!re.IsCompleted)
{
folderbutton.Enabled = false;
filebutton.Enabled = false;
numTextBox.Enabled = false;
searchbutton.Enabled = false;
renamebutton.Enabled = false;
}
//searchSpider.RenameFileName();
longAction.EndInvoke(re);
folderbutton.Enabled = true;
filebutton.Enabled = true;
numTextBox.Enabled = true;
searchbutton.Enabled = true;
renamebutton.Enabled = true;
MessageBox.Show("啦啦啦，终于做完了...");
}
else
{
MessageBox.Show("请选择要修改的文件夹");
}
}
catch
{
MessageBox.Show("请输入数字阈值");
}
}
static void Main(string[] args)
{
var scraper = new Scraper();
var spider = new Spider();
scraper.Links = spider.GetLinks("discover/advanced?category_id=12&woe_id=0&sort=most_funded");
scraper.Links = scraper.Links;//.Take(10); //Todo: add top x option
scraper.Download();
Console.WriteLine("Downloaded and scraped:");
foreach (var project in scraper.Projects)
{
Console.WriteLine(project.Name);
}
Console.WriteLine("Please Enter Output file name:");
var filename = Console.ReadLine();
scraper.Save(new XML(), filename + ".xml");
scraper.Save(new CSV(), filename + ".csv");
}
public void ProcessException()
{
var site = new Site {
EncodingName = "UTF-8", RemoveOutboundLinks = true
};
var scheduler = new QueueDuplicateRemovedScheduler();
site.AddStartUrl("http://v.youku.com/v_show/id_XMTMyMTkzNTY1Mg==.html?spm=a2h1n.8251845.0.0");
site.AddStartUrl("http://v.youku.com/v_show/id_XMjkzNzMwMDMyOA==.html?spm=a2h1n.8251845.0.0");
site.AddStartUrl("http://v.youku.com/v_show/id_XMjcwNDg0NDI3Mg==.html?spm=a2h1n.8251845.0.0");
site.AddStartUrl("http://v.youku.com/v_show/id_XMTMwNzQwMTcwMA==.html?spm=a2h1n.8251845.0.0");
site.AddStartUrl("http://v.youku.com/v_show/id_XMjk1MzI0Mzk4NA==.html?spm=a2h1n.8251845.0.0");
site.AddStartUrl("http://v.youku.com/v_show/id_XMjkzNzY0NzkyOA==.html?spm=a2h1n.8251845.0.0");
site.AddStartUrl("http://www.cnblogs.com/");
Spider spider = Spider.Create(site,
// crawler identity
"cnblogs_" + DateTime.Now.ToString("yyyyMMddhhmmss"),
// use memoery queue scheduler
scheduler,
// default page processor will save whole html, and extract urls to target urls via regex
new TestPageProcessor())
// save crawler result to file in the folder: \{running directory}\data\{crawler identity}\{guid}.dsd
.AddPipeline(new FilePipeline());
// dowload html by http client
spider.Downloader = new HttpClientDownloader();
spider.ThreadNum = 1;
// traversal deep 遍历深度
spider.Deep = 3;
// start crawler 启动爬虫
spider.Run();
}
private void InitializeApiObjects()
{
//New API needs to be instantiated here, in the same alphabetical order as above
acsrf = new Acsrf(this);
ajaxspider = new AjaxSpider(this);
alert = new OWASPZAPDotNetAPI.Generated.Alert(this);
alertFilter = new AlertFilter(this);
ascan = new Ascan(this);
authentication = new Authentication(this);
authorization = new OWASPZAPDotNetAPI.Generated.Authorization(this);
autoupdate = new Autoupdate(this);
brk = new Break(this);
context = new Context(this);
core = new Core(this);
forcedUser = new ForcedUser(this);
httpSessions = new HttpSessions(this);
importLogFiles = new ImportLogFiles(this);
importurls = new Importurls(this);
localProxies = new LocalProxies(this);
openapi = new Openapi(this);
parameters = new Params(this);
pnh = new Pnh(this);
pscan = new Pscan(this);
replacer = new Replacer(this);
reveal = new Reveal(this);
ruleConfig = new RuleConfig(this);
script = new Script(this);
search = new Search(this);
selenium = new Selenium(this);
sessionManagement = new SessionManagement(this);
soap = new Soap(this);
spider = new Spider(this);
stats = new Stats(this);
users = new Users(this);
websocket = new Websocket(this);
}
public static void StartAndWait(Frontier frontier, Index index, Filtering.Filter filter, int pagecount)
{
int count = (int)Math.Ceiling(pagecount / (double)SPIDER_PAGE_COUNT);
Spider[] spiders = new Spider[count];
Thread[] threads = new Thread[count];
for (int i = 0; i < count; i++)
{
Spider sp = spiders[i] = new Spider(frontier, index, filter, ind =>
{
Console.ForegroundColor = ConsoleColor.Cyan;
Console.WriteLine("Merging Index of {0}", ind.SiteCount);
Console.ForegroundColor = ConsoleColor.Gray;
lock (index) { index.MergeIn(ind); }
});
threads[i] = new Thread(() => sp.Run());
threads[i].Start();
}
for (int i = 0; i < count; i++)
threads[i].Join();
}
private void searchbutton_Click(object sender, EventArgs e)
{
try
{
int num = Int32.Parse(numTextBox.Text);
if (filelabel.Text != "" && filelabel.Text != "待测文档位置" &&
folderlabel.Text != "" && folderlabel.Text != "语料库文件夹路径")
{
Spider searchSpider = new Spider(filelabel.Text, folderlabel.Text, num);
Action longAction = new Action(searchSpider.Search);
IAsyncResult re = longAction.BeginInvoke(null, null);
while (!re.IsCompleted)
{
folderbutton.Enabled = false;
filebutton.Enabled = false;
numTextBox.Enabled = false;
searchbutton.Enabled = false;
}
//dsearchSpider.Search();
longAction.EndInvoke(re);
folderbutton.Enabled = true;
filebutton.Enabled = true;
numTextBox.Enabled = true;
searchbutton.Enabled = true;
MessageBox.Show("啦啦啦，终于做完了...");
}
else
{
MessageBox.Show("请补充完整待测文档及语料库位置");
}
}
catch
{
MessageBox.Show("请输入数字阈值");
}
}
public override void Handle(ref Page page, ISpider spider)
{
if (page == null || string.IsNullOrEmpty(ExceptionMessage) || string.IsNullOrEmpty(page.Content))
{
return;
}
if (page.Exception != null)
{
if (string.IsNullOrEmpty(ExceptionMessage))
{
page.Exception = new SpiderException("ExceptionMessage should not be empty/null.");
}
if (page.Exception.Message.Contains(ExceptionMessage))
{
if (NetworkCenter.Current.Executor.Redial() == RedialResult.Failed)
{
Logger.MyLog(spider.Identity, "Exit program because redial failed.", LogLevel.Error);
spider.Exit();
}
Spider.AddToCycleRetry(page.Request, spider.Site);
page.Exception = new DownloadException("Download failed and redial finished already.");
}
}
}
private static void Chasing()
{
if (Utils.SleepCheck("All"))
{
var me = ObjectMgr.LocalHero;
var target = me.ClosestToMouseTarget(1500);
var Spiderlings = ObjectMgr.GetEntities <Unit>().Where(spiderlings => spiderlings.ClassID == ClassID.CDOTA_Unit_Broodmother_Spiderling).ToList();
{
if (target != null && target.IsAlive && !target.IsIllusion && chasing.isActive)
{
{
foreach (var Spider in Spiderlings)
{
if (Spider.Distance2D(target) <= 1000 && (Utils.SleepCheck("combo")))
{
Spider.Attack(target);
}
}
Utils.Sleep(750, "combo");
}
}
else
{
if (Utils.SleepCheck("combo"))
{
foreach (var Spider in Spiderlings)
{
Spider.Move(Game.MousePosition);
}
}
Utils.Sleep(750, "combo");
}
}
}
Utils.Sleep(350, "All");
}
public static void Run()
{
// Config encoding, header, cookie, proxy etc... 定义采集的 Site 对象, 设置 Header、Cookie、代理等
var site = new Site {
EncodingName = "UTF-8"
};
for (int i = 1; i < 5; ++i)
{
// Add start/feed urls. 添加初始采集链接
site.AddStartUrl($"http://list.youku.com/category/show/c_96_s_1_d_1_p_{i}.html");
}
Spider spider = Spider.Create(site,
// use memoery queue scheduler. 使用内存调度
new QueueDuplicateRemovedScheduler(),
// use custmize processor for youku 为优酷自定义的 Processor
new YoukuPageProcessor())
// use custmize pipeline for youku 为优酷自定义的 Pipeline
.AddPipeline(new YoukuPipeline());
// Start crawler 启动爬虫
spider.Run();
}
private void InitializeApiObjects()
{
//New API needs to be instantiated here, in the same alphabetical order as above
Acsrf = new Acsrf(this);
AjaxSpider = new AjaxSpider(this);
Alert = new Generated.Alert(this);
AlertFilter = new AlertFilter(this);
Ascan = new Ascan(this);
Authentication = new Authentication(this);
Authorization = new Authorization(this);
AutoUpdate = new AutoUpdate(this);
Brk = new Break(this);
Context = new Context(this);
Core = new Core(this);
ForcedUser = new ForcedUser(this);
HttpSessions = new HttpSessions(this);
ImportLogFiles = new ImportLogFiles(this);
ImportUrls = new ImportUrls(this);
LocalProxies = new LocalProxies(this);
OpenApi = new OpenApi(this);
Parameters = new Params(this);
Pnh = new Pnh(this);
PScan = new PScan(this);
Replacer = new Replacer(this);
Reveal = new Reveal(this);
RuleConfig = new RuleConfig(this);
Script = new Script(this);
Search = new Search(this);
Selenium = new Selenium(this);
SessionManagement = new SessionManagement(this);
Soap = new Soap(this);
Spider = new Spider(this);
Stats = new Stats(this);
Users = new Users(this);
Websocket = new Websocket(this);
}
public virtual void Run(params string[] args)
{
if (string.IsNullOrEmpty(Name) || Name.Length > 120)
{
throw new ArgumentException("Length of name should between 1 and 120.");
}
if (string.IsNullOrEmpty(ConnectString))
{
ConnectString = Core.Infrastructure.Configuration.GetValue(Core.Infrastructure.Configuration.LogAndStatusConnectString);
}
Spider = GetEntitySpider();
if (Spider == null)
{
throw new SpiderException("Spider is null.");
}
if (!string.IsNullOrEmpty(ConnectString))
{
PrepareDb();
InsertTask();
InsertBatch();
}
Spider.Run(args);
}
public static void ChasingAll(EventArgs args)
{
var me = ObjectMgr.LocalHero;
if (!Game.IsInGame || me.ClassID != ClassID.CDOTA_Unit_Hero_Broodmother || me == null)
{
return;
}
var target = me.ClosestToMouseTarget(1900);
if (activCombo && target != null && target.IsAlive && !target.IsIllusion)
{
var Spiderlings = ObjectMgr.GetEntities <Unit>().Where(spiderlings => spiderlings.ClassID == ClassID.CDOTA_Unit_Broodmother_Spiderling).ToList();
foreach (var Spider in Spiderlings)
{
if (Spider.Distance2D(target) <= 1500 && Utils.SleepCheck(Spider.Handle.ToString()))
{
Spider.Attack(target);
Utils.Sleep(900, Spider.Handle.ToString());
}
}
foreach (var Spider in Spiderlings)
{
if (Spider.Distance2D(target) >= 1500 && Utils.SleepCheck(Spider.Handle.ToString()))
{
Spider.Move(Game.MousePosition);
Utils.Sleep(900, Spider.Handle.ToString());
}
}
var linkens = target.Modifiers.Any(x => x.Name == "modifier_item_spheretarget") || target.Inventory.Items.Any(x => x.Name == "item_sphere");
var enemies = ObjectMgr.GetEntities <Hero>().Where(hero => hero.IsAlive && !hero.IsIllusion && hero.IsVisible && hero.Team != me.Team).ToList();
if (target != null && target.IsAlive && !target.IsIllusion && me.Distance2D(target) <= 1000)
{
if (Q == null)
{
Q = me.Spellbook.SpellQ;
}
if (W == null) ///////It will be added later//////////
{
W = me.Spellbook.SpellW;
}
if (R == null)
{
R = me.Spellbook.SpellR;
}
// Item
if (sheep == null)
{
sheep = target.ClassID == ClassID.CDOTA_Unit_Hero_Tidehunter ? null : me.FindItem("item_sheepstick");
}
if (cheese == null)
{
cheese = me.FindItem("item_cheese");
}
if (orchid == null)
{
orchid = me.FindItem("item_orchid");
}
if (Soul == null)
{
Soul = me.FindItem("item_soul_ring");
}
if (shiva == null)
{
shiva = me.FindItem("item_shivas_guard");
}
if (dagon == null)
{
dagon = me.Inventory.Items.FirstOrDefault(item => item.Name.Contains("item_dagon"));
}
if (mom == null)
{
mom = me.FindItem("item_mask_of_madness");
}
if (abyssal == null)
{
abyssal = me.FindItem("item_abyssal_blade");
}
if (mjollnir == null)
{
mjollnir = me.FindItem("item_mjollnir");
}
if (halberd == null)
{
halberd = me.FindItem("item_heavens_halberd");
}
if (medall == null)
{
medall = me.FindItem("item_medallion_of_courage") ?? me.FindItem("item_solar_crest");
}
if (satanic == null)
{
satanic = me.FindItem("item_satanic");
}
if ( // Q Skill
Q != null &&
Q.CanBeCasted() &&
me.CanCast() &&
!target.IsMagicImmune() &&
me.Distance2D(target) <= 600 &&
Utils.SleepCheck("Q")
)
{
Q.UseAbility(target);
Utils.Sleep(250 + Game.Ping, "Q");
} // Q Skill end
if ( //R Skill
R != null &&
R.CanBeCasted() &&
me.CanCast() &&
me.Distance2D(target) <= 350 &&
Utils.SleepCheck("R")
)
{
R.UseAbility();
Utils.Sleep(250 + Game.Ping, "R");
} // R Skill end
if ( // orchid
orchid != null &&
orchid.CanBeCasted() &&
me.CanCast() &&
!target.IsMagicImmune() &&
!linkens &&
Utils.SleepCheck("orchid") &&
me.Distance2D(target) <= 1000
)
{
orchid.UseAbility(target);
Utils.Sleep(250 + Game.Ping, "orchid");
} // orchid Item end
if ( // sheep
sheep != null &&
sheep.CanBeCasted() &&
me.CanCast() &&
!target.IsMagicImmune() &&
!linkens &&
Utils.SleepCheck("sheep") &&
me.Distance2D(target) <= 600
)
{
sheep.UseAbility(target);
Utils.Sleep(250 + Game.Ping, "sheep");
} // sheep Item end
if ( // Soul Item
Soul != null &&
me.Health / me.MaximumHealth <= 0.5 &&
me.Mana <= Q.ManaCost &&
Soul.CanBeCasted())
{
Soul.UseAbility();
} // Soul Item end
if ( // Shiva Item
shiva != null &&
shiva.CanBeCasted() &&
me.CanCast() &&
!target.IsMagicImmune() &&
Utils.SleepCheck("shiva") &&
me.Distance2D(target) <= 600
)
{
shiva.UseAbility();
Utils.Sleep(250 + Game.Ping, "shiva");
} // Shiva Item end
if ( // MOM
mom != null &&
mom.CanBeCasted() &&
me.CanCast() &&
Utils.SleepCheck("mom") &&
me.Distance2D(target) <= 700
)
{
mom.UseAbility();
Utils.Sleep(250 + Game.Ping, "mom");
} // MOM Item end
if ( // Medall
medall != null &&
medall.CanBeCasted() &&
Utils.SleepCheck("Medall") &&
me.Distance2D(target) <= 500
)
{
medall.UseAbility(target);
Utils.Sleep(250 + Game.Ping, "Medall");
} // Medall Item end
if ( // Abyssal Blade
abyssal != null &&
abyssal.CanBeCasted() &&
me.CanCast() &&
!target.IsMagicImmune() &&
Utils.SleepCheck("abyssal") &&
me.Distance2D(target) <= 400
)
{
abyssal.UseAbility(target);
Utils.Sleep(250 + Game.Ping, "abyssal");
} // Abyssal Item end
if ( // Hellbard
halberd != null &&
halberd.CanBeCasted() &&
me.CanCast() &&
!target.IsMagicImmune() &&
Utils.SleepCheck("halberd") &&
me.Distance2D(target) <= 700
)
{
halberd.UseAbility(target);
Utils.Sleep(250 + Game.Ping, "halberd");
} // Hellbard Item end
if ( // Mjollnir
mjollnir != null &&
mjollnir.CanBeCasted() &&
me.CanCast() &&
!target.IsMagicImmune() &&
Utils.SleepCheck("mjollnir") &&
me.Distance2D(target) <= 600
)
{
mjollnir.UseAbility(me);
Utils.Sleep(250 + Game.Ping, "mjollnir");
} // Mjollnir Item end
if ( // Dagon
dagon != null &&
dagon.CanBeCasted() &&
me.CanCast() &&
!target.IsMagicImmune() &&
Utils.SleepCheck("dagon")
)
{
dagon.UseAbility(target);
Utils.Sleep(250 + Game.Ping, "dagon");
} // Dagon Item end
if ( // Satanic
satanic != null &&
me.Health / me.MaximumHealth <= 0.4 &&
satanic.CanBeCasted() &&
me.Distance2D(target) <= 300
&&
Utils.SleepCheck("Satanic")
)
{
satanic.UseAbility();
Utils.Sleep(250 + Game.Ping, "Satanic");
} // Satanic Item end
if ( //Attack
me.Distance2D(target) <= 1900 &&
Utils.SleepCheck("Attack")
)
{
me.Attack(target);
Utils.Sleep(300 + Game.Ping, "Attack");
} // Attack
/***************************************WWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWW**********************************/
List <Unit> SpideWeb = ObjectMgr.GetEntities <Unit>().Where(web => web.ClassID == ClassID.CDOTA_Unit_Broodmother_Web).ToList();
e = ObjectMgr.GetEntities <Hero>()
.Where(x => x.IsAlive && x.Team != me.Team && !x.IsIllusion)
.OrderBy(x => x.Position.Distance2D(SpideWeb.OrderBy(y => x.Position.Distance2D(y.Position)).FirstOrDefault().Position))
.FirstOrDefault();
if (e.Distance2D(SpideWeb.FirstOrDefault()) > 1100 && e != null && W != null && e.IsAlive && !e.IsIllusion)
{
if (e.Distance2D(SpideWeb.FirstOrDefault()) >= 1100 && me.Distance2D(e) <= 600 && Utils.SleepCheck(e.Handle.ToString()) && W.CanBeCasted())
{
W.UseAbility(e.Position);
Utils.Sleep(4000, e.Handle.ToString());
}
}
/***************************************WWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWW**********************************/
}
}
}
/// <summary>
/// Setup this workload manager for the specified spider.
/// This method is not used by the MemoryWorkloadManager.
/// </summary>
/// <param name="spider">The spider using this workload manager.</param>
public void Init(Spider spider)
{
}
public abstract void Build(Spider site, dynamic obj);
public bool ReorderTracks(Spider.Media.Playlist playlist, int startPos, int[] tracks, int newPos)
{
throw new NotImplementedException();
}
public void Play(Spider.Media.Track track)
{
throw new NotImplementedException();
}
public Spider.Media.TrackCollection LoadTracksForGivenRelease(Spider.Media.Release release)
{
throw new NotImplementedException();
}
private void btnDownloadSite_Click(object sender, EventArgs e)
{
ResourceLocation rl = this.location1.ResourceLocation;
rl.BindProtocolProviderType();
if (rl.ProtocolProviderType == null)
{
MessageBox.Show("Invalid URL format, please check the location field.",
AppManager.Instance.Application.MainForm.Text, MessageBoxButtons.OK, MessageBoxIcon.Error);
return;
}
AutoDownloadsExtension scheduler;
scheduler = (AutoDownloadsExtension)AppManager.Instance.Application.GetExtensionByType(typeof(AutoDownloadsExtension));
if (! scheduler.Active)
{
using (StartAutoDownloadsForm startAd = new StartAutoDownloadsForm())
{
if (startAd.ShowDialog() == DialogResult.Yes)
{
startAd.ScheduledDownloadEnabler.ApplySettings();
}
}
if (!scheduler.Active)
{
return;
}
}
SpiderContext spiderCntx = new SpiderContext(
folderBrowser1.Folder,
rl.URL);
spiderCntx.NamingRules.Add(
new BaseLocationNamingRule(chkImagesFromExternal.Checked, chkResourcesFromExternal.Checked));
if (chkExcludeExtensions.Checked)
{
spiderCntx.NamingRules.Add(new ExcludeExtensionNamingRule(txtExcludeExt.Text.Split(',')));
}
if (chkExcludeNames.Checked)
{
spiderCntx.NamingRules.Add(new ExcludeNameNamingRule(txtExcludeNames.Text.Split(',')));
}
if (numDownloadDepth.Value > 0)
{
spiderCntx.NamingRules.Add(new MaxDepthNamingRule((int)numDownloadDepth.Value));
}
if (chkRemoveCompleted.Checked)
{
spiderCntx.ResourceCompleted += delegate(SpiderContext cntx, ISpiderResource resource)
{
DownloadManager.Instance.ClearEnded();
};
}
siteSpider = new Spider(spiderCntx, new WebSpiderResourceFactory());
siteSpider.Completed += new CompletedEventHandler(siteSpider_Completed);
siteSpider.Aborted += new EventHandler(siteSpider_Aborted);
siteSpider.BeginLookup();
EnterDownloadMode(true);
}
public Spider.Media.ReleaseCollection LoadReleasesForGivenArtist(Spider.Media.Artist artist, Spider.Media.ReleaseType type, int page)
{
throw new NotImplementedException();
}
public bool InsertTrack(Spider.Media.Playlist playlist, Spider.Media.Track track, int pos)
{
throw new NotImplementedException();
}
public Spider.Media.TrackCollection GetPlaylistTracks(Spider.Media.Playlist playlist, int revision)
{
throw new NotImplementedException();
}
public bool DeleteTracks(Spider.Media.Playlist playlist, int[] indexes)
{
throw new NotImplementedException();
}
public void HandleTagEvent(Uri baseUri, Spider.Engine.Tag tag)
{
// IMPLEMENT ME!
}
/// <summary>
/// Setup this workload manager for the specified spider.
/// </summary>
/// <param name="spider">The spider using this workload manager.</param>
public virtual void Init(Spider spider)
{
this.spider = spider;
this.addLock = new Semaphore(1, 1);
this.workLatch = new AutoResetEvent(false);
this.connectionString = spider.Options.DbConnectionString;
this.statements.Add(this.stmtClear = new RepeatableStatement(this.holder.getSQLClear()));
this.statements.Add(this.stmtClear2 = new RepeatableStatement(this.holder.getSQLClear2()));
this.statements.Add(this.stmtAdd = new RepeatableStatement(this.holder.getSQLAdd()));
this.statements.Add(this.stmtAdd2 = new RepeatableStatement(this.holder.getSQLAdd2()));
this.statements.Add(this.stmtGetWork = new RepeatableStatement(this.holder.getSQLGetWork()));
this.statements.Add(this.stmtGetWork2 = new RepeatableStatement(this.holder.getSQLGetWork2()));
this.statements
.Add(this.stmtWorkloadEmpty = new RepeatableStatement(this.holder.getSQLWorkloadEmpty()));
this.statements.Add(this.stmtSetWorkloadStatus = new RepeatableStatement(
this.holder.getSQLSetWorkloadStatus()));
this.statements.Add(this.stmtSetWorkloadStatus2 = new RepeatableStatement(
this.holder.getSQLSetWorkloadStatus2()));
this.statements.Add(this.stmtGetDepth = new RepeatableStatement(this.holder.getSQLGetDepth()));
this.statements.Add(this.stmtGetSource = new RepeatableStatement(this.holder.getSQLGetSource()));
this.statements.Add(this.stmtResume = new RepeatableStatement(this.holder.getSQLResume()));
this.statements.Add(this.stmtResume2 = new RepeatableStatement(this.holder.getSQLResume2()));
this.statements
.Add(this.stmtGetWorkloadID = new RepeatableStatement(this.holder.getSQLGetWorkloadID()));
this.statements.Add(this.stmtGetHostID = new RepeatableStatement(this.holder.getSQLGetHostID()));
this.statements.Add(this.stmtGetNextHost = new RepeatableStatement(this.holder.getSQLGetNextHost()));
this.statements
.Add(this.stmtSetHostStatus = new RepeatableStatement(this.holder.getSQLSetHostStatus()));
this.statements.Add(this.stmtGetHost = new RepeatableStatement(this.holder.getSQLGetHost()));
try
{
Open();
this.maxURLSize = GetColumnSize("spider_workload", "url");
this.maxHostSize = GetColumnSize("spider_host", "host");
}
catch (Exception e)
{
throw (new WorkloadException(e));
}
}
private static void Spider_OnClosed(Spider spider)
{
Console.WriteLine($"Spider: {spider.Identity} closed, Status: {spider.Stat}");
}
public Spider.Media.TopList LoadTopListForResource(Spider.Media.Resource res)
{
throw new NotImplementedException();
}
public SpiderCombo(List<ButtonsManager.Button> bl, Spider sp) {
buttonsList = bl;
spider = sp;
}
public void Start()
{
_spider = transform.parent.GetComponent <Spider>();
}
public void SpiderTest()
{
var spider = new Spider();
var links = spider.GetLinks("/discover/categories/video%20games/most-funded");
}