/// <summary>
/// Crawls a site.
/// </summary>
public int CrawlSite(Site site)
{
_site = site;
// get the robot helper class up and running
_robot = Robots.Load( _site.IndexStartingLocation );
_startUrl = _site.IndexStartingLocation;
CrawlPage( _site.IndexStartingLocation );
return _pages.Count;
}
/// <summary>
/// Returns Site object from cache. If site does not already exist in cache, it
/// will be read and added to cache
/// </summary>
/// <param name="id"></param>
/// <returns></returns>
public static Site Read( int id )
{
string cacheKey = Site.CacheKey( id );
ObjectCache cache = MemoryCache.Default;
Site site = cache[cacheKey] as Site;
if ( site != null )
return site;
else
{
Rock.CMS.SiteService siteService = new CMS.SiteService();
Rock.CMS.Site siteModel = siteService.Get( id );
if ( siteModel != null )
{
site = new Site();
site.Id = siteModel.Id;
site.Name = siteModel.Name;
site.Description = siteModel.Description;
site.Theme = siteModel.Theme;
site.DefaultPageId = siteModel.DefaultPageId;
site.AppleTouchUrl = siteModel.AppleTouchIconUrl;
site.FaviconUrl = siteModel.FaviconUrl;
site.FacebookAppId = siteModel.FacebookAppId;
site.FacebookAppSecret = siteModel.FacebookAppSecret;
Rock.Attribute.Helper.LoadAttributes( siteModel );
foreach ( var category in siteModel.Attributes )
foreach ( var attribute in category.Value )
site.AttributeIds.Add( attribute.Id );
site.AttributeValues = siteModel.AttributeValues;
site.AuthEntity = siteModel.AuthEntity;
site.SupportedActions = siteModel.SupportedActions;
cache.Set( cacheKey, site, new CacheItemPolicy() );
return site;
}
else
return null;
}
}
/// <summary>
/// Initializes a new instance of the <see cref="Crawler"/> class.
/// </summary>
/// <param name="site">The site.</param>
public Crawler( Site site )
{
this.CrawlSite( site );
}
/// <summary>
/// Job that will run quick SQL queries on a schedule.
///
/// Called by the <see cref="IScheduler" /> when a
/// <see cref="ITrigger" /> fires that is associated with
/// the <see cref="IJob" />.
/// </summary>
public virtual void Execute( IJobExecutionContext context )
{
JobDataMap dataMap = context.JobDetail.JobDataMap;
var siteId = dataMap.GetString( "Site" ).AsIntegerOrNull();
Uri startingUri;
if ( siteId.HasValue )
{
_site = new SiteService( new RockContext()).Get( siteId.Value );
if ( _site != null )
{
var startingUrl = _site.IndexStartingLocation;
if ( Uri.TryCreate( startingUrl, UriKind.Absolute, out startingUri ) && (startingUri.Scheme == Uri.UriSchemeHttp || startingUri.Scheme == Uri.UriSchemeHttps) )
{
// ensure that an index is configured for site pages, if not create it
IndexContainer.CreateIndex( typeof( SitePageIndex ), false );
// release the crawler, like the kraken... but not...
var pages = new Crawler().CrawlSite( _site );
context.Result = string.Format( "Crawler found {0} pages, {1} pages sent to be indexed.", pages, _indexedPageCount );
}
else
{
context.Result = "An invalid starting URL was provided.";
}
}
else
{
context.Result = "Could not locate the site provided.";
}
}
else
{
context.Result = "An invalid site was provided.";
}
}
