/// <summary> /// Constructs a new instance of the <see cref="CWClientScheduler"/> class. /// </summary> public CWClientScheduler() { // This call is required by the Windows.Forms Component Designer. InitializeComponent(); globals = Globals.Instance(); tmrStart = new System.Timers.Timer(); startTimer = new AlarmTimer(); if(globals.Settings.EnableScheduler) { startTimer.AlarmTime = globals.Settings.StartTime.AddMinutes(-5); } else { startTimer.AlarmTime = DateTime.Now.AddMinutes(1); } startTimer.Enabled = true; startTimer.OnAlarmBell += new EventHandler(startTimer_OnAlarmBell); startTimer.Start(); updateBackoff = new Backoff(BackoffSpeed.Linear); updating = false; }
/// <summary> /// Create a new instance of the <see cref="WordExtractorPlugin"/> class. /// </summary> public WordExtractorPlugin() { mutex = new Mutex(); settings = PluginSettings.Instance(); name = "CrawlWave.ServerPlugins.WordExtraction"; description = "CrawlWave Word Extraction Plugin"; dataDependent = false; state = PluginState.Stopped; enabled = true; version = Assembly.GetExecutingAssembly().GetName().Version.ToString(); percent = 0; mustStop = false; pluginThread = null; if(settings.UseDatabase) { dbProvider = DBConnectionStringProvider.Instance(); settings.DBConnectionString = dbProvider.ProvideDBConnectionString(name); dbcon = new SqlConnection(settings.DBConnectionString); } wordExtractor = WordExtractor.Instance(); cache = WordsCache.Instance(); backoff = new Backoff(BackoffSpeed.Slow, 30000); }
/// <summary> /// Constructs a new istance of the <see cref="Crawler"/> class and initializes its /// properties with the default values. The constructor is private so that only the /// class itself can create an instance. /// </summary> private Crawler() { //first of all get a reference to the global variables because they are needed //in order to initialize some variables. globals = Globals.Instance(); mustStop = false; stopping = false; state = CrawlerState.Stopped; stats = new long[10] {0,0,0,0,0,0,0,0,0,0}; numThreads = (int)globals.Settings.ConnectionSpeed; runningThreads = 0; //sendResultsThread = null; synchronizeThread = null; crawlingThreads = null; syncBackOff = new Backoff(BackoffSpeed.Declining, 30000); downloadBackOff = new Backoff(BackoffSpeed.Fast); urlsToCrawl = new Queue(); resultFileNames = new Queue(); crawledUrls = new ArrayList(); queueSize = 0; dataFileName = String.Empty; defaultEncoding = Encoding.GetEncoding("ISO-8859-7"); defaultGreekEncoding = Encoding.GetEncoding(1253); contentRegex = new Regex("<meta\\s*http-equiv=([^>])*charset\\s*=\\s*([^>])*(utf-7|utf-8|utf-16|windows-1253)([^>])*>",RegexOptions.CultureInvariant|RegexOptions.Multiline|RegexOptions.IgnoreCase|RegexOptions.Compiled); htmlParser = HtmlParser.Instance(); textParser = TextParser.Instance(); pdfParser = PdfParser.Instance(); swfParser = SwfParser.Instance(); nullParser = NullParser.Instance(); robotsFilter = RobotsFilter.Instance(); robotsFilter.LoadEntries(); domainFilter = DomainFilter.Instance(); hostRequestFilter = HostRequestFilter.Instance(); hostBanFilter = HostBanFilter.Instance(); //proxy = WebServiceProxy.Instance(); proxy = CrawlWaveServerProxy.Instance(globals); }