private void AddItem(DomainFilter Filter) { var Item = new FilterListItem(Filter); Item.PropertyChanged += Item_PropertyChanged; FilterList.Add(Item); }
static void Main(string[] args) { Console.WindowWidth += 50; frontier = new Frontier(new Exclusions()); frontier.Add(new URL("http://en.wikipedia.org/wiki/Teenage_Mutant_Ninja_Turtles")); Filter filter = new DomainFilter("en.wikipedia.org") & new ExtentionFilter(false, "jpg", "jpeg", "gif", "png", "rar", "zip", "exe", "pdf"); DateTime start = DateTime.Now; Crawler.StartAndWait(frontier, index, filter, 100); DateTime end = DateTime.Now; Console.WriteLine("Crawler done in {0:0.00} sec ({1:0.00} pages per sec).", (end - start).TotalSeconds, index.SiteCount / (end - start).TotalSeconds); Console.WriteLine("Press any key to start querying."); Console.ReadKey(true); Console.WriteLine(); start = DateTime.Now; Ranker r = new Ranker(index, TrimmingStemmer.GetStemmer(PorterStemmer.StemTerm)); end = DateTime.Now; Console.WriteLine("Ranker created in {0:0.00} sec.", (end - start).TotalSeconds); string searchQuery = ""; while (true) { Console.WriteLine("Query for data below. Enter an empty string to quit."); Console.Write("Search for: "); searchQuery = Console.ReadLine(); if (searchQuery == "") { break; } start = DateTime.Now; foreach (var doc in r.GetHits(searchQuery).OrderByDescending(x => x.Item2)) { Console.WriteLine("Rank: {1:0.000000} for: {0}", doc.Item1.URL.Address, doc.Item2); } end = DateTime.Now; Console.WriteLine("Query completed in {0:0.00} sec", (end - start).TotalSeconds); Console.WriteLine(); } frontier.Kill(); }
private void AddDomain(string Domain, bool RegExp) { if (RegExp ? MiscFunc.IsValidRegex(Domain) : Uri.CheckHostName(Domain.Replace("*", "asterisk")) != UriHostNameType.Dns) { MessageBox.Show(Translate.fmt("msg_bad_dns_filter"), App.Title, MessageBoxButton.OK, MessageBoxImage.Stop); return; } // don't add duplicated foreach (var Item in FilterList) { if (Item.Filter.Domain.Equals(Domain)) { MessageBox.Show(Translate.fmt("msg_dns_filter_dup"), App.Title, MessageBoxButton.OK, MessageBoxImage.Exclamation); return; } } DomainFilter Filter = new DomainFilter() { Domain = Domain }; if (RegExp) { Filter.Format = DomainFilter.Formats.RegExp; } else if (Domain.Contains("*")) { Filter.Format = DomainFilter.Formats.WildCard; } AddItem(Filter); App.client.UpdateDomainFilter(ListType, Filter); txtDomain.Text = ""; }
public Downloader(Uri uri, string downloadPath, DomainFilter filterFlag = DomainFilter.None, int maxLevel = 0, params string[] fileExtenthionFilter) { if (ReferenceEquals(uri, null)) { throw new ArgumentNullException(nameof(uri), "Input parameter can not be null"); } if (ReferenceEquals(downloadPath, null)) { throw new ArgumentNullException(nameof(downloadPath), "Input parameter can not be null"); } startUri = uri; this.downloadPath = downloadPath; if (!Directory.Exists(this.downloadPath)) { Directory.CreateDirectory(this.downloadPath); } this.maxLevel = maxLevel; this.filterFlag = filterFlag; this.fileExtenthionFilter = new ConcurrentHashSet <string>(fileExtenthionFilter); }
public FilterListItem(DomainFilter Filter) { this.Filter = Filter; }
public Downloader(Uri uri, string downloadPath, DomainFilter filterFlag, params string[] fileExtenthionFilter) : this(uri, downloadPath, filterFlag, 0, fileExtenthionFilter) { }