public SpiderContext ToRuntimeContext() { SpiderContext context = new SpiderContext(); context.CachedSize = CachedSize; context.PageHandlers = GetCustomziePage(PageHandlers); context.TargetUrlsHandler = GetCustomizeTargetUrls(TargetUrlsHandler); context.Deep = Deep; context.Downloader = GetDownloader(Downloader); context.EmptySleepTime = EmptySleepTime; context.Entities = Entities; context.SkipWhenResultIsEmpty = SkipWhenResultIsEmpty; context.Pipelines = GetPipepines(Pipelines); context.PrepareStartUrls = GetPrepareStartUrls(PrepareStartUrls); context.Redialer = GetRedialer(Redialer); context.Scheduler = GetScheduler(Scheduler); context.Site = Site; context.StartUrls = StartUrls; context.SpiderName = SpiderName; context.ThreadNum = ThreadNum; context.EnviromentValues = EnviromentValues; context.Validations = GetValidations(Validations); context.UserId = UserId; context.TaskGroup = TaskGroup; context.TargetUrlExtractInfos = GetTargetUrlExtractInfos(TargetUrlExtractInfos); return(context); }
protected override SpiderContext GetSpiderContext() { SpiderContext context = new SpiderContext(); context.SetTaskGroup("JD sku/store test"); context.SetSpiderName("JD sku/store test " + DateTime.Now.ToString("yyyy-MM-dd HHmmss")); context.AddTargetUrlExtractor(new Extension.Configuration.TargetUrlExtractor { Region = new Extension.Configuration.Selector { Type = ExtractType.XPath, Expression = "//span[@class=\"p-num\"]" }, Patterns = new List <string> { @"&page=[0-9]+&" } }); context.AddPipeline(new MysqlPipeline { ConnectString = "Database='test';Data Source=ooodata.com;User ID=root;Password=1qazZAQ!123456;Port=4306" }); context.SetScheduler(new RedisScheduler { Host = "ooodata.com", Password = "******", Port = 6379 }); context.AddStartUrl("http://list.jd.com/list.html?cat=9987,653,655&page=2&JL=6_0_0&ms=5#J_main", new Dictionary <string, object> { { "name", "手机" }, { "cat3", "655" } }); context.AddEntityType(typeof(Product)); return(context); }
public bool Accept(Uri location, SpiderContext context, ISpiderResource parentSpider) { UrlType urlType = ((WebSpiderResource)parentSpider).addingUrlType; if ( !(urlType == UrlType.Img && this.downloadImagesFromExternalSite) && !(urlType != UrlType.Img && this.downloadResourceFromExternalSite) ) { string tempbase = context.BaseLocation; int i; if ((i = tempbase.LastIndexOf("/")) >= 0) { tempbase = tempbase.Substring(0, i + 1); } if (!location.OriginalString.StartsWith(tempbase)) { return(false); } } return(true); }
protected override SpiderContext GetSpiderContext() { SpiderContext context = new SpiderContext(); context.SetSpiderName("ddeng.com " + DateTime.Now.ToString("yyyy-MM-dd HHmmss")); context.SetSite(new Site { Cookie = "sid=dea284fc36c24e8cbcd447343d7b8a4e; sn=DD962248; ctid=000000; ctnm=%E5%8F%A4%E9%95%87%E7%81%AF%E9%A5%B0%E6%89%B9%E5%8F%91; ctpv=%E5%B9%BF%E4%B8%9C; JSESSIONID=acbBqFfOD4I63d9PziDvv; DDENG=c4fc08ae2e3ba3efeddbc667c2f45e615a85e80009169501dc244a03e87908aa61146548b97ed9c7dc07af23bfd80bff5008f8c8867a9165d4bd2732aca0db7dedae2e042d3968fcad1150f36be242e8a32a3f59db2a0b39216a59f1628508c5799644532a9d99925f9841b3c13a1f97; userId=10003379; previousUser=%E5%A4%95%E7%8E%89; Hm_lvt_9e33f153f28be198970d205d90a24f28=1466146335; Hm_lpvt_9e33f153f28be198970d205d90a24f28=1466146392; Hm_lvt_54b4cb498afd05463ab4611b38a6f289=1466146335; Hm_lpvt_54b4cb498afd05463ab4611b38a6f289=1466146392; CNZZDATA1256982382=395301521-1466143554-%7C1466143554", Headers = new Dictionary <string, string> { { "Cache-Control", "max-age=0" }, { "Upgrade-Insecure-Requests", "1" } }, UserAgent = "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.102 Safari/537.36", Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8" }); context.AddPipeline(new MysqlPipeline { ConnectString = "Database='test';Data Source=;User ID=root;Password=;Port=4306" }); context.AddStartUrl("http://www.ddeng.com/product/967659"); context.AddEntityType(typeof(Corp)); return(context); }
public bool Accept(Uri location, SpiderContext context, ISpiderResource parentSpider) { UrlType urlType = ((WebSpiderResource)parentSpider).addingUrlType; if ( !(urlType == UrlType.Img && this.downloadImagesFromExternalSite) && !(urlType != UrlType.Img && this.downloadResourceFromExternalSite) ) { string tempbase = context.BaseLocation; int i; if ((i = tempbase.LastIndexOf("/")) >= 0) { tempbase = tempbase.Substring(0, i + 1); } if (!location.OriginalString.StartsWith(tempbase)) { return false; } } return true; }
public static void Main(string[] args) { Core.Spider.PrintInfo(); //Options param = ParseCommand(args); //if (param != null) //{ // StartSpider(param); //} string json = File.ReadAllText("mysqlsample.json"); json = Macros.Replace(json); SpiderContext spiderContext = JsonConvert.DeserializeObject <SpiderContext>(json); List <string> errorMessages; if (SpiderContextValidation.Validate(spiderContext, out errorMessages)) { ScriptSpider spider = new ScriptSpider(spiderContext); spider.Run(args); } else { foreach (var errorMessage in errorMessages) { Console.WriteLine(errorMessage); } } Console.Read(); }
protected override SpiderContext GetSpiderContext() { SpiderContext context = new SpiderContext(); context.SetSpiderName("JD sku/store test " + DateTime.Now.ToString("yyyy-MM-dd HHmmss")); context.AddTargetUrlExtractor(new Extension.Configuration.TargetUrlExtractor { Region = new Extension.Configuration.Selector { Type = ExtractType.XPath, Expression = "//span[@class=\"p-num\"]" }, Patterns = new List <string> { @"&page=[0-9]+&" } }); context.AddPipeline(new MysqlPipeline { ConnectString = "Database='mysql';Data Source=192.168.199.211;User ID=root;Password=1qazZAQ!;Port=3306" }); context.AddStartUrl("http://list.jd.com/list.html?cat=9987,653,655&page=2&JL=6_0_0&ms=5#J_main", new Dictionary <string, object> { { "name", "手机" }, { "cat3", "655" } }); context.AddEntityType(typeof(Product)); context.SetDownloader(new WebDriverDownloader { Browser = Extension.Downloader.WebDriver.Browser.Chrome }); return(context); }
protected override SpiderContext GetSpiderContext() { SpiderContext context = new SpiderContext(); context.Site = new Site { MaxSleepTime = 1, MinSleepTime = 1 }; context.SetTaskGroup("cnblogs homepage"); context.SetSpiderName("cnblogs homepage " + DateTime.Now.ToString("yyyy-MM-dd HHmmss")); context.AddStartUrl("http://news.cnblogs.com/n/page/1/"); context.AddPipeline(new MysqlPipeline { ConnectString = "Database='taobao';Data Source= 86research.imwork.net;User ID=root;Password=1qazZAQ!;Port=4306" }); context.AddTargetUrlExtractor(new Extension.Configuration.TargetUrlExtractor { Region = new Extension.Configuration.Selector { Expression = "//*[@id='pager']", Type = ExtractType.XPath } }); context.ThreadNum = 5; context.AddEntityType(typeof(Cnblogs)); return(context); }
protected override SpiderContext GetSpiderContext() { SpiderContext context = new SpiderContext(); context.SetThreadNum(2); context.SetSpiderName("JD_sku_store_test_" + DateTime.Now.ToString("yyyy_MM_dd_HHmmss")); context.AddTargetUrlExtractor(new Extension.Configuration.TargetUrlExtractor { Region = new Extension.Configuration.Selector { Type = ExtractType.XPath, Expression = "//span[@class=\"p-num\"]" }, Patterns = new List <string> { @"&page=[0-9]+&" } }); context.AddPipeline(new MysqlPipeline { ConnectString = "Database='test';Data Source=mysqlserver;User ID=root;Password=1qazZAQ!;Port=4306" }); context.AddStartUrl("http://list.jd.com/list.html?cat=9987,653,655&page=2&JL=6_0_0&ms=5#J_main", new Dictionary <string, object> { { "name", "手机" }, { "cat3", "655" } }); context.AddEntityType(typeof(Product)); return(context); }
public bool Accept(Uri location, SpiderContext context, ISpiderResource parentSpider) { string ext = Path.GetExtension(location.Segments[location.Segments.Length - 1]).ToLower(); int index = Array.IndexOf <string>(this.extensionsToExclude, ext); return(index < 0); }
public bool TodayIsGetProduct(string RefCatalogLink) { using (DbContext Context = new SpiderContext()) { string rankTime = DateTime.Now.ToString("yyyy-MM-dd"); return(Context.Set <Catalog>().Count(o => o.CatalogLink == RefCatalogLink && o.RankTime == rankTime) > 0); } }
public void Add(T entity) { using (DbContext Context = new SpiderContext()) { Context.Set <T>().Add(entity); Save(Context); } }
public bool Accept(Uri location, SpiderContext context, ISpiderResource parentSpider) { string ext = Path.GetExtension(location.Segments[location.Segments.Length-1]).ToLower(); int index = Array.IndexOf<string>(this.extensionsToExclude, ext); return index < 0; }
static async Task DownloadImages() { Func <DownloadModel, Task <string> > asyncTask = async model => { using (WebClient webClient = new WebClient()) { Console.WriteLine($"下载文件{model.Url}"); string fileName = Guid.NewGuid().ToString().Replace("-", "") + ".jpg"; string path = Path.Combine(model.alias, fileName); try { string savePath = Path.Combine(BasePath, model.alias); if (!Directory.Exists(savePath)) { Directory.CreateDirectory(savePath); } await webClient.DownloadFileTaskAsync(model.Url, Path.Combine(savePath, fileName)); Console.WriteLine($"下载完成,输出路径{path}"); return(path); } catch (Exception e) { Console.WriteLine($"下载出错:{e.Message}"); return(""); } } }; Func <DownloadModel, Task> saveRecordAsync = async model => { Console.WriteLine("准备保存地址到数据库"); using (SpiderContext db = new SpiderContext()) { var article = await db.WeChatArticles.FirstOrDefaultAsync(c => c.Id == model.Id); if (null != article) { article.Download = true; article.LocalPath = model.Path; await db.SaveChangesAsync(); } } Console.WriteLine("保存完成"); }; foreach (var item in downloadModels) { item.Path = await asyncTask(item); if (!string.IsNullOrWhiteSpace(item.Path)) { await saveRecordAsync(item); } Console.WriteLine($"在此休眠{SleepTime / 1000}秒"); Thread.Sleep(SleepTime); } }
public LinkContextSpider(SpiderContext spiderContext) : base(spiderContext) { var nextSpider = spiderContext as LinkSpiderContext; if (nextSpider != null) { NextSpiders = nextSpider.NextSpiderContexts; } }
public void Start() { var scheduler = new SchedulerTest(); var context = new SpiderContext("https://www.baidu.com/", ""); var spider = new SpiderOperater(scheduler, "обть╡Бйт", context); spider.Start(); Assert.AreEqual(true, context.Result.Succeeded); }
public async Task <List <WeChatAccount> > ListWeChatAccountAsync() { using var db = new SpiderContext(); return(await db.WeChatAccounts .Include(c => c.TaskStartSign) .Include(role => role.SpiderRoles) .OrderBy(c => c.Id) .ToListAsync()); }
protected override SpiderContext GetSpiderContext() { SpiderContext context = new SpiderContext(); context.SetTaskGroup("cnblogs homepage"); context.SetSpiderName("cnblogs homepage " + DateTime.Now.ToString("yyyy-MM-dd HHmmss")); context.AddStartUrl("http://www.cnblogs.com"); context.AddPipeline(new ConslePipeline()); context.AddEntityType(typeof(HomePage)); return context; }
protected override Task SuccessPipelineAsync(SpiderContext context) { locations.Add(Tuple.Create <Tuple <double, double>, string>(Tuple.Create <double, double>(context.Data.latitude, context.Data.longitude), context.Data.name)); //Console.WriteLine(context.Data.latitude); //Console.WriteLine(context.Data.longitude); //Console.WriteLine(context.Data.name); //Console.WriteLine("-------------------------------"); return(base.SuccessPipelineAsync(context)); }
public bool Accept(Uri location, SpiderContext context, ISpiderResource parentSpider) { if (this.maxDepth > 0 && parentSpider.Depth >= this.maxDepth) { return false; } else { return true; } }
protected override SpiderContext GetSpiderContext() { SpiderContext context = new SpiderContext(); context.SetTaskGroup("cnblogs homepage"); context.SetSpiderName("cnblogs homepage " + DateTime.Now.ToString("yyyy-MM-dd HHmmss")); context.AddStartUrl("http://www.cnblogs.com"); context.AddPipeline(new ConslePipeline()); context.AddEntityType(typeof(HomePage)); return(context); }
public bool Accept(Uri location, SpiderContext context, ISpiderResource parentSpider) { if (this.maxDepth > 0 && parentSpider.Depth >= this.maxDepth) { return(false); } else { return(true); } }
public void DeleteProduct(string RefCatalogLink, string RankTime) { using (DbContext Context = new SpiderContext()) { SQLiteParameter[] sqlParameter = new SQLiteParameter[] { new SQLiteParameter("@RefCatalogLink", RefCatalogLink), new SQLiteParameter("@RankTime", RankTime) }; Context.Database.ExecuteSqlCommand($"DELETE FROM Product WHERE RefCatalogLink=@RefCatalogLink AND RankTime=@RankTime; UPDATE Catalog SET RankTime=@RankTime WHERE CatalogLink=@RefCatalogLink;", sqlParameter); } }
public void AddRange(List <Product> list) { using (DbContext Context = new SpiderContext()) { list.ForEach(l => { l.ID = Guid.NewGuid().ToString().ToUpper(); }); Context.Set <Product>().AddRange(list); Save(Context); } }
public async Task <int> SetAccountSpiderStop(long accountId) { using var spiderContext = new SpiderContext(); var account = await spiderContext.WeChatAccounts.FirstOrDefaultAsync(c => c.Id == accountId); if (null == account) { throw new NullReferenceException("未找到公众号实体"); } account.TaskStartSign = null; return(await spiderContext.SaveChangesAsync()); }
public async Task <int> SetClientSignOut(long clientId) { using var spiderContext = new SpiderContext(); var sign = await spiderContext.TaskStartSigns.FirstOrDefaultAsync(c => c.ClientId == clientId); if (null == sign) { throw new NullReferenceException("未找到实体"); } sign.RunStatus = false; return(await spiderContext.SaveChangesAsync()); }
public async Task InsertClientSgin(long clientId) { using var db = new SpiderContext(); db.TaskStartSigns.Add(new TaskStartSign { Id = IdWorkContext.ID_WORKER.NextId(), ClientId = clientId, StartDate = DateTime.Now, RunStatus = true }); await db.SaveChangesAsync(); }
public void AddRange(List <Product> list) { using (DbContext Context = new SpiderContext()) { DbSet <Product> dbSetEntity = Context.Set <Product>(); Product first = list[0]; list.ForEach(l => { l.ID = Guid.NewGuid(); }); dbSetEntity.AddRange(list); Save(Context); } }
public bool Accept(Uri uri, SpiderContext context, ISpiderResource parentSpider) { string fileName = uri.Segments[uri.Segments.Length - 1]; for (int i = 0; i < namesToExclude.Length; i++) { if (fileName.IndexOf(namesToExclude[i]) >= 0) { return false; } } return true; }
public bool Accept(Uri uri, SpiderContext context, ISpiderResource parentSpider) { string fileName = uri.Segments[uri.Segments.Length - 1]; for (int i = 0; i < namesToExclude.Length; i++) { if (fileName.IndexOf(namesToExclude[i]) >= 0) { return(false); } } return(true); }
protected override SpiderContext GetSpiderContext() { SpiderContext context = new SpiderContext(); context.SetTaskGroup("JD sku/store test"); context.SetSpiderName("JD sku/store test " + DateTime.Now.ToString("yyyy-MM-dd HHmmss")); context.AddTargetUrlExtractor(new Extension.Configuration.TargetUrlExtractor { Region = new Extension.Configuration.Selector { Type = ExtractType.XPath, Expression = "//span[@class=\"p-num\"]" }, Patterns = new List <string> { @"&page=[0-9]+&" } }); context.AddPipeline(new MysqlPipeline { ConnectString = "Database='test';Data Source=ooodata.com;User ID=root;Password=1qazZAQ!123456;Port=4306" }); context.SetScheduler(new RedisScheduler { Host = "ooodata.com", Password = "******", Port = 6379 }); context.AddStartUrl("http://list.jd.com/list.html?cat=9987,653,655&page=1", new Dictionary <string, object> { { "name", "手机" }, { "cat3", "655" } }); context.AddPrepareStartUrls(new ConfigurableDbPrepareStartUrls() { Source = DataSource.MySql, ConnectString = "Database='test';Data Source= ooodata.com;User ID=root;Password=1qazZAQ!123456;Port=4306", TableName = "jd.category", Columns = new List <BaseDbPrepareStartUrls.Column> { new BaseDbPrepareStartUrls.Column { Name = "url", Formatters = new List <Formatter> { new ReplaceFormatter { OldValue = ".html", NewValue = "" } } } }, FormateStrings = new List <string> { "{0}&page=1&JL=6_0_0" } }); context.AddEntityType(typeof(Product)); return(context); }
public Catalog AddIfNoExist(Catalog entity) { using (DbContext db = new SpiderContext()) { var findEntity = db.Set <Catalog>().Where(o => o.RID == entity.RID).FirstOrDefault(); if (findEntity != null) { return(findEntity); } entity.ID = Guid.NewGuid().ToString().ToUpper(); db.Set <Catalog>().Add(entity); Save(db); return(entity); } }
public async Task UpdateClientSgin(long clientId) { using var spiderContext = new SpiderContext(); var sign = await spiderContext.TaskStartSigns.FirstOrDefaultAsync(c => c.ClientId == clientId); if (null != sign) { sign.StartDate = DateTime.Now; await spiderContext.SaveChangesAsync(); } else { throw new NullReferenceException("未找到实体"); } }
public async Task <int> InsertArticleByList(List <WeChatArticle> list, long accountId) { using var db = new SpiderContext(); var account = await db.WeChatAccounts.FirstOrDefaultAsync(c => c.Id == accountId); if (null == account) { throw new NullReferenceException("未找到公众号实体"); } list.ForEach(item => { item.WeChatAccount = account; db.WeChatArticles.Add(item); }); return(await db.SaveChangesAsync()); }
public Catalog AddIfNoExist(Catalog entity) { using (DbContext db = new SpiderContext()) { var findEntity = db.Set <Catalog>().Where(o => o.CatalogLink == entity.CatalogLink).FirstOrDefault(); if (findEntity != null) { return(findEntity); } entity.ID = Guid.NewGuid(); entity.CreateTime = DateTime.Now; db.Set <Catalog>().Add(entity); Save(db); return(entity); } }
protected override SpiderContext GetSpiderContext() { SpiderContext context = new SpiderContext(); context.SetSpiderName("JD sku/store test " + DateTime.Now.ToString("yyyy-MM-dd HHmmss")); context.AddTargetUrlExtractor(new Extension.Configuration.TargetUrlExtractor { Region = new Extension.Configuration.Selector { Type = ExtractType.XPath, Expression = "//span[@class=\"p-num\"]" }, Patterns = new List<string> { @"&page=[0-9]+&" } }); context.AddPipeline(new MysqlPipeline { ConnectString = "Database='test';Data Source=;User ID=root;Password=;Port=4306" }); context.AddStartUrl("http://list.jd.com/list.html?cat=9987,653,655&page=2&JL=6_0_0&ms=5#J_main", new Dictionary<string, object> { { "name", "手机" }, { "cat3", "655" } }); context.AddEntityType(typeof(Product)); return context; }
public ModelSpider(SpiderContext spiderContext) { #if NET_CORE Encoding.RegisterProvider(CodePagesEncodingProvider.Instance); #endif SpiderContext = spiderContext; if (!SpiderContext.IsBuilt) { SpiderContext.Build(); } Name = $"{SpiderContext.UserId}-{SpiderContext.SpiderName}"; Logger = LogUtils.GetLogger(SpiderContext.SpiderName, SpiderContext.UserId, SpiderContext.TaskGroup); InitEnvoriment(); }
public WebSpiderResource(SpiderContext context, ISpiderResource parent, string location) { this.context = context; this.location = location; this.parent = parent; UpdateDepth(); string localFile = GetLocalFile(); download = DownloadManager.Instance.Add( ResourceLocation.FromURL(this.Location), null, localFile, 1, false); download.StateChanged += new EventHandler(download_StateChanged); }
public WebSpiderResource(SpiderContext context, ISpiderResource parent, string location) { this.context = context; this.location = location; this.parent = parent; UpdateDepth(); string localFile = GetLocalFile(); download = DownloadManager.Instance.Add( ResourceLocation.FromURL(this.Location), null, localFile, 1, false); download.StateChanged += new EventHandler(download_StateChanged); }
protected override SpiderContext GetSpiderContext() { SpiderContext context = new SpiderContext(); context.SetSpiderName("ddeng.com " + DateTime.Now.ToString("yyyy-MM-dd HHmmss")); context.SetSite(new Site { Cookie = "sid=dea284fc36c24e8cbcd447343d7b8a4e; sn=DD962248; ctid=000000; ctnm=%E5%8F%A4%E9%95%87%E7%81%AF%E9%A5%B0%E6%89%B9%E5%8F%91; ctpv=%E5%B9%BF%E4%B8%9C; JSESSIONID=acbBqFfOD4I63d9PziDvv; DDENG=c4fc08ae2e3ba3efeddbc667c2f45e615a85e80009169501dc244a03e87908aa61146548b97ed9c7dc07af23bfd80bff5008f8c8867a9165d4bd2732aca0db7dedae2e042d3968fcad1150f36be242e8a32a3f59db2a0b39216a59f1628508c5799644532a9d99925f9841b3c13a1f97; userId=10003379; previousUser=%E5%A4%95%E7%8E%89; Hm_lvt_9e33f153f28be198970d205d90a24f28=1466146335; Hm_lpvt_9e33f153f28be198970d205d90a24f28=1466146392; Hm_lvt_54b4cb498afd05463ab4611b38a6f289=1466146335; Hm_lpvt_54b4cb498afd05463ab4611b38a6f289=1466146392; CNZZDATA1256982382=395301521-1466143554-%7C1466143554", Headers = new Dictionary<string, string> { { "Cache-Control","max-age=0"}, { "Upgrade-Insecure-Requests","1"} }, UserAgent = "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.102 Safari/537.36", Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8" }); context.AddPipeline(new MysqlPipeline { ConnectString = "Database='test';Data Source=;User ID=root;Password=;Port=4306" }); context.AddStartUrl("http://www.ddeng.com/product/967659"); context.AddEntityType(typeof(Corp)); return context; }
private void btnDownloadSite_Click(object sender, EventArgs e) { ResourceLocation rl = this.location1.ResourceLocation; rl.BindProtocolProviderType(); if (rl.ProtocolProviderType == null) { MessageBox.Show("Invalid URL format, please check the location field.", AppManager.Instance.Application.MainForm.Text, MessageBoxButtons.OK, MessageBoxIcon.Error); return; } AutoDownloadsExtension scheduler; scheduler = (AutoDownloadsExtension)AppManager.Instance.Application.GetExtensionByType(typeof(AutoDownloadsExtension)); if (! scheduler.Active) { using (StartAutoDownloadsForm startAd = new StartAutoDownloadsForm()) { if (startAd.ShowDialog() == DialogResult.Yes) { startAd.ScheduledDownloadEnabler.ApplySettings(); } } if (!scheduler.Active) { return; } } SpiderContext spiderCntx = new SpiderContext( folderBrowser1.Folder, rl.URL); spiderCntx.NamingRules.Add( new BaseLocationNamingRule(chkImagesFromExternal.Checked, chkResourcesFromExternal.Checked)); if (chkExcludeExtensions.Checked) { spiderCntx.NamingRules.Add(new ExcludeExtensionNamingRule(txtExcludeExt.Text.Split(','))); } if (chkExcludeNames.Checked) { spiderCntx.NamingRules.Add(new ExcludeNameNamingRule(txtExcludeNames.Text.Split(','))); } if (numDownloadDepth.Value > 0) { spiderCntx.NamingRules.Add(new MaxDepthNamingRule((int)numDownloadDepth.Value)); } if (chkRemoveCompleted.Checked) { spiderCntx.ResourceCompleted += delegate(SpiderContext cntx, ISpiderResource resource) { DownloadManager.Instance.ClearEnded(); }; } siteSpider = new Spider(spiderCntx, new WebSpiderResourceFactory()); siteSpider.Completed += new CompletedEventHandler(siteSpider_Completed); siteSpider.Aborted += new EventHandler(siteSpider_Aborted); siteSpider.BeginLookup(); EnterDownloadMode(true); }
void siteSpider_Completed(SpiderContext cntx) { this.BeginInvoke((MethodInvoker)delegate() { EnterDownloadMode(false); }); }
public ISpiderResource CreateSpider(SpiderContext cntx, ISpiderResource parent, string location) { return new WebSpiderResource(cntx, parent, location); }
public EntityProcessor(SpiderContext spiderContext) { Site = spiderContext.Site; _spiderContext = spiderContext; }