/// <summary>
        ///     Processes the crawl request.
        /// </summary>
        /// <param name = "crawlRequest">The crawl request.</param>
        /// <param name = "fileManager">The file manager.</param>
        /// <param name = "imageManager">The image manager.</param>
        /// <param name = "webPageManager">The web page manager.</param>
        /// <param name = "arachnodeDAO">The arachnode DAO.</param>
        public override void ProcessCrawlRequest(CrawlRequest <TArachnodeDAO> crawlRequest, FileManager <TArachnodeDAO> fileManager, ImageManager <TArachnodeDAO> imageManager, WebPageManager <TArachnodeDAO> webPageManager, IArachnodeDAO arachnodeDAO)
        {
            switch (crawlRequest.DataType.DiscoveryType)
            {
            case DiscoveryType.File:
                ProcessFile(crawlRequest, fileManager, arachnodeDAO);
                break;

            case DiscoveryType.Image:
                ProcessImage(crawlRequest, imageManager, arachnodeDAO);
                break;

            case DiscoveryType.WebPage:
                if (crawlRequest.Discovery.ExpectFileOrImage || _discoveryManager.IsCrawlRestricted(crawlRequest, crawlRequest.Parent.Uri.AbsoluteUri))
                {
                    /*<img src="http://msn.com"></a> would return a WebPage.  If the Crawl was restricting crawling to fark.com, and a
                     * page on fark.com listed the aforementioned Image and we didn't make this check, then the Crawl would create a
                     * record in the WebPages table, which would not be correct.
                     * A CurrentDepth of '0' means a request was made for a File or an Image.
                     */
                    //ANODET: Improve parsing... check the regular expressions...
                    crawlRequest.Crawl.UnassignedDiscoveries.Remove(crawlRequest.Discovery.Uri.AbsoluteUri);

                    return;

#if !DEMO
                    //throw new Exception("A CrawlRequest was created for a File or an Image, but the HttpWebResponse returned a WebPage.  This is typically indicative of invalid HTML.");
#endif
                }
                ProcessWebPage(crawlRequest, webPageManager, arachnodeDAO);
                break;

#if !DEMO
            case DiscoveryType.None:

                if (ApplicationSettings.InsertDisallowedAbsoluteUris)
                {
                    arachnodeDAO.InsertDisallowedAbsoluteUri(crawlRequest.DataType.ContentTypeID, (int)crawlRequest.DataType.DiscoveryType, crawlRequest.Parent.Uri.AbsoluteUri, crawlRequest.Discovery.Uri.AbsoluteUri, "Disallowed by unassigned DataType.  (" + crawlRequest.DataType.ContentType + ")", ApplicationSettings.ClassifyAbsoluteUris);
                }

                break;
#endif
            }

            //remove the reference from the crawl.
            crawlRequest.Crawl.UnassignedDiscoveries.Remove(crawlRequest.Discovery.Uri.AbsoluteUri);
        }
        /// <summary>
        ///     Processes the image.
        /// </summary>
        /// <param name = "crawlRequest">The crawl request.</param>
        /// <param name = "imageManager">The image manager.</param>
        /// <param name = "arachnodeDAO">The arachnode DAO.</param>
        protected override void ProcessImage(CrawlRequest <TArachnodeDAO> crawlRequest, ImageManager <TArachnodeDAO> imageManager, IArachnodeDAO arachnodeDAO)
        {
            _consoleManager.OutputImageDiscovered(crawlRequest.Crawl.CrawlInfo.ThreadNumber, crawlRequest, crawlRequest.Discovery);
            Counters.GetInstance().ImagesDiscovered(1);

            if (crawlRequest.Discovery.DiscoveryState == DiscoveryState.PostRequest)
            {
                imageManager.ManageImage(crawlRequest);
            }
            else
            {
                if (ApplicationSettings.InsertImageDiscoveries)
                {
                    arachnodeDAO.InsertImageDiscovery(crawlRequest.Parent.Uri.AbsoluteUri, crawlRequest.Discovery.Uri.AbsoluteUri);
                }

                _consoleManager.OutputCacheHit(crawlRequest.Crawl.CrawlInfo, crawlRequest, crawlRequest.Discovery);
            }
        }