public void TestCreateNamedQueue()
        {
            MacroscopeNamedQueue <MacroscopeJobItem> NamedQueue = new MacroscopeNamedQueue <MacroscopeJobItem>();

            NamedQueue.CreateNamedQueue(QUEUENAME);
            MacroscopeJobItem JobItem1 = new MacroscopeJobItem("http://www.company.com/");

            NamedQueue.AddToNamedQueue(QUEUENAME, JobItem1);
            Assert.AreEqual(1, NamedQueue.CountNamedQueueItems(QUEUENAME));
        }
        public void TestFailAddingDuplicateToNamedQueue()
        {
            MacroscopeNamedQueue <MacroscopeJobItem> NamedQueue = new MacroscopeNamedQueue <MacroscopeJobItem>();

            NamedQueue.CreateNamedQueue(QUEUENAME, MacroscopeNamedQueue <MacroscopeJobItem> .MODE.USE_HISTORY);
            MacroscopeJobItem JobItem1 = new MacroscopeJobItem("http://www.company.com/");
            MacroscopeJobItem JobItem2 = new MacroscopeJobItem("http://www.company.com/");

            NamedQueue.AddToNamedQueue(QUEUENAME, JobItem1);
            NamedQueue.AddToNamedQueue(QUEUENAME, JobItem2);
            Assert.AreEqual(1, NamedQueue.CountNamedQueueItems(QUEUENAME));
        }
예제 #3
0
        /** -------------------------------------------------------------------- **/

        public Queue <T> CreateNamedQueue(string Name, MacroscopeNamedQueue <T> .MODE QueueMode)
        {
            Queue <T> NamedQueue = this.CreateNamedQueue(Name: Name);

            this.NamedQueuesMode[Name] = QueueMode;

            lock (this.NamedQueues)
            {
                lock (this.NamedQueues[Name])
                {
                    Dictionary <string, bool> NamedQueueHistory = new Dictionary <string, bool>(4096);

                    this.NamedQueuesHistory.Add(Name, NamedQueueHistory);
                }
            }

            return(NamedQueue);
        }
        public void TestAddRemoveAddAgainToNamedQueue()
        {
            MacroscopeNamedQueue <MacroscopeJobItem> NamedQueue = new MacroscopeNamedQueue <MacroscopeJobItem>();

            NamedQueue.CreateNamedQueue(QUEUENAME, MacroscopeNamedQueue <MacroscopeJobItem> .MODE.USE_HISTORY);

            MacroscopeJobItem JobItem1 = new MacroscopeJobItem("http://www.company.com/");

            NamedQueue.AddToNamedQueue(QUEUENAME, JobItem1);

            Assert.AreEqual(1, NamedQueue.CountNamedQueueItems(QUEUENAME));

            MacroscopeJobItem JobItem2 = NamedQueue.GetNamedQueueItem(QUEUENAME);

            Assert.AreEqual(0, NamedQueue.CountNamedQueueItems(QUEUENAME));

            bool Forgotten = NamedQueue.ForgetNamedQueueItem(QUEUENAME, JobItem1);

            NamedQueue.AddToNamedQueue(QUEUENAME, JobItem1);

            Assert.AreEqual(1, NamedQueue.CountNamedQueueItems(QUEUENAME));
        }
예제 #5
0
        /**************************************************************************/

        private void InitializeJobMaster(MacroscopeConstants.RunTimeMode JobRunTimeMode)
        {
            GC.Collect();

            /*
             * {
             * this.JobMasterLog = new EventLog ();
             * this.JobMasterLog.Source = MacroscopeConstants.MainEventLogSourceName;
             * this.JobGuid = Guid.NewGuid();
             * this.LogEntry( string.Format( "Starting Job" ) );
             * }
             */

            this.RunTimeMode = JobRunTimeMode;

            if (this.TaskController != null)
            {
                this.CredentialsHttp = this.TaskController.IGetCredentialsHttp();
            }

            this.DocCollection = new MacroscopeDocumentCollection(JobMaster: this);
            this.AllowedHosts  = new MacroscopeAllowedHosts();

            /** BEGIN: Named Queues *************************************************/

            this.NamedQueueJobItems = new MacroscopeNamedQueue <MacroscopeJobItem> ();

            this.NamedQueueJobItems.CreateNamedQueue(
                Name: MacroscopeConstants.NamedQueueUrlList,
                QueueMode: MacroscopeNamedQueue <MacroscopeJobItem> .MODE.USE_HISTORY
                );

            this.NamedQueue = new MacroscopeNamedQueue <string> ();

            {
                this.NamedQueue.CreateNamedQueue(Name: MacroscopeConstants.NamedQueueDisplayQueue);

                this.NamedQueue.CreateNamedQueue(Name: MacroscopeConstants.NamedQueueDisplayStructure);
                this.NamedQueue.CreateNamedQueue(Name: MacroscopeConstants.NamedQueueDisplayStructureLinkCounts);

                this.NamedQueue.CreateNamedQueue(Name: MacroscopeConstants.NamedQueueDisplayHierarchy);
                this.NamedQueue.CreateNamedQueue(Name: MacroscopeConstants.NamedQueueDisplayCanonicalAnalysis);
                this.NamedQueue.CreateNamedQueue(Name: MacroscopeConstants.NamedQueueDisplayHrefLang);
                this.NamedQueue.CreateNamedQueue(Name: MacroscopeConstants.NamedQueueDisplayErrors);
                this.NamedQueue.CreateNamedQueue(Name: MacroscopeConstants.NamedQueueDisplayHostnames);
                this.NamedQueue.CreateNamedQueue(Name: MacroscopeConstants.NamedQueueDisplayRedirectsAudit);
                this.NamedQueue.CreateNamedQueue(Name: MacroscopeConstants.NamedQueueDisplayLinks);
                this.NamedQueue.CreateNamedQueue(Name: MacroscopeConstants.NamedQueueDisplayHyperlinks);
                this.NamedQueue.CreateNamedQueue(Name: MacroscopeConstants.NamedQueueDisplayUriAnalysis);
                this.NamedQueue.CreateNamedQueue(Name: MacroscopeConstants.NamedQueueDisplayPageTitles);
                this.NamedQueue.CreateNamedQueue(Name: MacroscopeConstants.NamedQueueDisplayPageDescriptions);
                this.NamedQueue.CreateNamedQueue(Name: MacroscopeConstants.NamedQueueDisplayPageKeywords);
                this.NamedQueue.CreateNamedQueue(Name: MacroscopeConstants.NamedQueueDisplayPageHeadings);
                this.NamedQueue.CreateNamedQueue(Name: MacroscopeConstants.NamedQueueDisplayPageText);
                this.NamedQueue.CreateNamedQueue(Name: MacroscopeConstants.NamedQueueDisplayStylesheets);
                this.NamedQueue.CreateNamedQueue(Name: MacroscopeConstants.NamedQueueDisplayImages);
                this.NamedQueue.CreateNamedQueue(Name: MacroscopeConstants.NamedQueueDisplayJavascripts);
                this.NamedQueue.CreateNamedQueue(Name: MacroscopeConstants.NamedQueueDisplayAudios);
                this.NamedQueue.CreateNamedQueue(Name: MacroscopeConstants.NamedQueueDisplayVideos);
                this.NamedQueue.CreateNamedQueue(Name: MacroscopeConstants.NamedQueueDisplaySitemaps);
                this.NamedQueue.CreateNamedQueue(Name: MacroscopeConstants.NamedQueueDisplayEmailAddresses);
                this.NamedQueue.CreateNamedQueue(Name: MacroscopeConstants.NamedQueueDisplayTelephoneNumbers);
                this.NamedQueue.CreateNamedQueue(Name: MacroscopeConstants.NamedQueueDisplayCustomFilters);

                this.NamedQueue.CreateNamedQueue(Name: MacroscopeConstants.NamedQueueDisplayDataExtractorsCssSelectors);
                this.NamedQueue.CreateNamedQueue(Name: MacroscopeConstants.NamedQueueDisplayDataExtractorsRegexes);
                this.NamedQueue.CreateNamedQueue(Name: MacroscopeConstants.NamedQueueDisplayDataExtractorsXpaths);

                this.NamedQueue.CreateNamedQueue(Name: MacroscopeConstants.NamedQueueDisplayRemarks);
            }

            /** END: Named Queues ***************************************************/

            this.CrawlDelay = 0;

            this.AdjustThreadsMax();
            this.ThreadsRunning = 0;
            this.ThreadsStop    = false;
            this.ThreadsDict    = new Dictionary <int, Boolean> ();

            this.SemaphoreWorkers = new Semaphore(0, this.ThreadsMax);
            this.SemaphoreWorkers.Release(this.ThreadsMax);

            this.Depth          = MacroscopePreferencesManager.GetDepth();
            this.PageLimit      = MacroscopePreferencesManager.GetPageLimit();
            this.PageLimitCount = 0;

            this.PagesFound = 0;

            {
                this.ParentStartingDirectory = "";
                this.ChildStartingDirectory  = "";
            }

            this.JobHistory = new MacroscopeJobHistory();

            this.InitProgress();

            this.Locales = new Dictionary <string, string> (32);

            this.Robots          = new MacroscopeRobots();
            this.BlockedByRobots = new Dictionary <string, Boolean> ();
        }