private void AddItem(DomainFilter Filter)
{
var Item = new FilterListItem(Filter);
Item.PropertyChanged += Item_PropertyChanged;
FilterList.Add(Item);
}
static void Main(string[] args)
{
Console.WindowWidth += 50;
frontier = new Frontier(new Exclusions());
frontier.Add(new URL("http://en.wikipedia.org/wiki/Teenage_Mutant_Ninja_Turtles"));
Filter filter = new DomainFilter("en.wikipedia.org") & new ExtentionFilter(false, "jpg", "jpeg", "gif", "png", "rar", "zip", "exe", "pdf");
DateTime start = DateTime.Now;
Crawler.StartAndWait(frontier, index, filter, 100);
DateTime end = DateTime.Now;
Console.WriteLine("Crawler done in {0:0.00} sec ({1:0.00} pages per sec).", (end - start).TotalSeconds, index.SiteCount / (end - start).TotalSeconds);
Console.WriteLine("Press any key to start querying.");
Console.ReadKey(true);
Console.WriteLine();
start = DateTime.Now;
Ranker r = new Ranker(index, TrimmingStemmer.GetStemmer(PorterStemmer.StemTerm));
end = DateTime.Now;
Console.WriteLine("Ranker created in {0:0.00} sec.", (end - start).TotalSeconds);
string searchQuery = "";
while (true)
{
Console.WriteLine("Query for data below. Enter an empty string to quit.");
Console.Write("Search for: ");
searchQuery = Console.ReadLine();
if (searchQuery == "")
{
break;
}
start = DateTime.Now;
foreach (var doc in r.GetHits(searchQuery).OrderByDescending(x => x.Item2))
{
Console.WriteLine("Rank: {1:0.000000} for: {0}", doc.Item1.URL.Address, doc.Item2);
}
end = DateTime.Now;
Console.WriteLine("Query completed in {0:0.00} sec", (end - start).TotalSeconds);
Console.WriteLine();
}
frontier.Kill();
}
private void AddDomain(string Domain, bool RegExp)
{
if (RegExp ? MiscFunc.IsValidRegex(Domain) : Uri.CheckHostName(Domain.Replace("*", "asterisk")) != UriHostNameType.Dns)
{
MessageBox.Show(Translate.fmt("msg_bad_dns_filter"), App.Title, MessageBoxButton.OK, MessageBoxImage.Stop);
return;
}
// don't add duplicated
foreach (var Item in FilterList)
{
if (Item.Filter.Domain.Equals(Domain))
{
MessageBox.Show(Translate.fmt("msg_dns_filter_dup"), App.Title, MessageBoxButton.OK, MessageBoxImage.Exclamation);
return;
}
}
DomainFilter Filter = new DomainFilter()
{
Domain = Domain
};
if (RegExp)
{
Filter.Format = DomainFilter.Formats.RegExp;
}
else if (Domain.Contains("*"))
{
Filter.Format = DomainFilter.Formats.WildCard;
}
AddItem(Filter);
App.client.UpdateDomainFilter(ListType, Filter);
txtDomain.Text = "";
}
public Downloader(Uri uri, string downloadPath, DomainFilter filterFlag = DomainFilter.None, int maxLevel = 0, params string[] fileExtenthionFilter)
{
if (ReferenceEquals(uri, null))
{
throw new ArgumentNullException(nameof(uri), "Input parameter can not be null");
}
if (ReferenceEquals(downloadPath, null))
{
throw new ArgumentNullException(nameof(downloadPath), "Input parameter can not be null");
}
startUri = uri;
this.downloadPath = downloadPath;
if (!Directory.Exists(this.downloadPath))
{
Directory.CreateDirectory(this.downloadPath);
}
this.maxLevel = maxLevel;
this.filterFlag = filterFlag;
this.fileExtenthionFilter = new ConcurrentHashSet <string>(fileExtenthionFilter);
}
public FilterListItem(DomainFilter Filter)
{
this.Filter = Filter;
}
public Downloader(Uri uri, string downloadPath, DomainFilter filterFlag, params string[] fileExtenthionFilter) : this(uri, downloadPath, filterFlag, 0, fileExtenthionFilter)
{
}