/// <summary> /// Crawls a site. /// </summary> public int CrawlSite(Site site) { _site = site; // get the robot helper class up and running _robot = Robots.Load( _site.IndexStartingLocation ); _startUrl = _site.IndexStartingLocation; CrawlPage( _site.IndexStartingLocation ); return _pages.Count; }
/// <summary> /// Returns Site object from cache. If site does not already exist in cache, it /// will be read and added to cache /// </summary> /// <param name="id"></param> /// <returns></returns> public static Site Read( int id ) { string cacheKey = Site.CacheKey( id ); ObjectCache cache = MemoryCache.Default; Site site = cache[cacheKey] as Site; if ( site != null ) return site; else { Rock.CMS.SiteService siteService = new CMS.SiteService(); Rock.CMS.Site siteModel = siteService.Get( id ); if ( siteModel != null ) { site = new Site(); site.Id = siteModel.Id; site.Name = siteModel.Name; site.Description = siteModel.Description; site.Theme = siteModel.Theme; site.DefaultPageId = siteModel.DefaultPageId; site.AppleTouchUrl = siteModel.AppleTouchIconUrl; site.FaviconUrl = siteModel.FaviconUrl; site.FacebookAppId = siteModel.FacebookAppId; site.FacebookAppSecret = siteModel.FacebookAppSecret; Rock.Attribute.Helper.LoadAttributes( siteModel ); foreach ( var category in siteModel.Attributes ) foreach ( var attribute in category.Value ) site.AttributeIds.Add( attribute.Id ); site.AttributeValues = siteModel.AttributeValues; site.AuthEntity = siteModel.AuthEntity; site.SupportedActions = siteModel.SupportedActions; cache.Set( cacheKey, site, new CacheItemPolicy() ); return site; } else return null; } }
/// <summary> /// Initializes a new instance of the <see cref="Crawler"/> class. /// </summary> /// <param name="site">The site.</param> public Crawler( Site site ) { this.CrawlSite( site ); }
/// <summary> /// Job that will run quick SQL queries on a schedule. /// /// Called by the <see cref="IScheduler" /> when a /// <see cref="ITrigger" /> fires that is associated with /// the <see cref="IJob" />. /// </summary> public virtual void Execute( IJobExecutionContext context ) { JobDataMap dataMap = context.JobDetail.JobDataMap; var siteId = dataMap.GetString( "Site" ).AsIntegerOrNull(); Uri startingUri; if ( siteId.HasValue ) { _site = new SiteService( new RockContext()).Get( siteId.Value ); if ( _site != null ) { var startingUrl = _site.IndexStartingLocation; if ( Uri.TryCreate( startingUrl, UriKind.Absolute, out startingUri ) && (startingUri.Scheme == Uri.UriSchemeHttp || startingUri.Scheme == Uri.UriSchemeHttps) ) { // ensure that an index is configured for site pages, if not create it IndexContainer.CreateIndex( typeof( SitePageIndex ), false ); // release the crawler, like the kraken... but not... var pages = new Crawler().CrawlSite( _site ); context.Result = string.Format( "Crawler found {0} pages, {1} pages sent to be indexed.", pages, _indexedPageCount ); } else { context.Result = "An invalid starting URL was provided."; } } else { context.Result = "Could not locate the site provided."; } } else { context.Result = "An invalid site was provided."; } }