private static void TestBloomFilter(FilterDbCollection collection, string domain, bool whitelisted)
{
Stopwatch sw = Stopwatch.StartNew();
bool result = collection.PrefetchIsDomainInList(domain, whitelisted);
sw.Stop();
long nsPerTick = (1000L * 1000L * 1000L) / Stopwatch.Frequency;
Console.WriteLine("Prefetch {0} took {1}ns. result = {2}", domain, sw.ElapsedTicks * nsPerTick, result);
}
private static void TestForBug20()
{
var col = new FilterDbCollection();
col.ParseStoreRules(new[] { "%3D$popup,domain=nme.com" }, 1);
col.FinalizeForRead();
var uri = new Uri("https://vsco.co/goodvibes-vsco/media/5c2fb09c8e529d3e52b4231b?share=MTU0NjYyOTI4MA%3D%3D");
var allRules = col.GetFiltersForDomain("nme.com");
var nvm = new NameValueCollection();
var len = allRules.Count();
foreach (var wlr in allRules)
{
Debug.Assert(wlr.IsMatch(uri, nvm) == false);
}
Console.ReadKey();
}
private static void TestDomainWideException()
{
FilterDbCollection col = new FilterDbCollection();
//"@@$third-party,referer=~pinterest.com"
col.ParseStoreRules(new[] { "@@$referer=pinterest.com" }, 1);
col.FinalizeForRead();
var headersShouldMatch = new NameValueCollection(StringComparer.OrdinalIgnoreCase)
{
{ "X-Requested-With", "XmlHttpRequest" },
{ "Content-Type", "script" },
{ "Referer", "https://www.pinterest.com" },
};
var headersShouldnt = new NameValueCollection(StringComparer.OrdinalIgnoreCase)
{
{ "X-Requested-With", "XmlHttpRequest" },
{ "Content-Type", "script" },
{ "Referer", "https://www.silsly.com" },
};
var uri = new Uri("http://silly.com/stoopid/url&=b1");
var allRules = col.GetWhitelistFiltersForDomain();
foreach (var wlr in allRules)
{
Console.WriteLine("Inc R: {0}", string.Join(", ", wlr.ApplicableReferers));
Console.WriteLine("Exc R: {0}", string.Join(", ", wlr.ExceptReferers));
Console.WriteLine(wlr.IsMatch(uri, headersShouldMatch));
Console.WriteLine(wlr.IsMatch(uri, headersShouldnt));
}
Console.ReadKey();
}
private static void Main(string[] args)
{
var parser = new AbpFormatRuleParser();
string easylistPath = AppDomain.CurrentDomain.BaseDirectory + "easylist.txt";
string adultDomainsPath = AppDomain.CurrentDomain.BaseDirectory + "adult_domains.txt";
var easylistLines = File.ReadAllLines(easylistPath);
var adultLines = File.ReadAllLines(adultDomainsPath);
var sw = new Stopwatch();
Console.WriteLine("Testing Parser Speed");
Console.WriteLine("About To Parse {0} Filters From Easylist", easylistLines.Length);
var compiledFilters = new List <Filter>(easylistLines.Length);
sw.Start();
foreach (var entry in easylistLines)
{
compiledFilters.Add(parser.ParseAbpFormattedRule(entry, 1));
}
sw.Stop();
Console.WriteLine("Parsed {0} Easylist filters in {1} msec, averaging {2} msec per filter.", compiledFilters.Count, sw.ElapsedMilliseconds, sw.ElapsedMilliseconds / (double)compiledFilters.Count);
compiledFilters = new List <Filter>(adultDomainsPath.Length);
Console.WriteLine();
Console.WriteLine("Testing Parser Speed");
Console.WriteLine("About To Parse {0} Filters From Adult Domains", adultLines.Length);
sw.Restart();
foreach (var entry in adultLines)
{
compiledFilters.Add(parser.ParseAbpFormattedRule(entry, 1));
}
sw.Stop();
Console.WriteLine("Parsed {0} Adult Domains filters in {1} msec, averaging {2} msec per filter.", compiledFilters.Count, sw.ElapsedMilliseconds, sw.ElapsedMilliseconds / (double)compiledFilters.Count);
Array.Clear(adultLines, 0, adultLines.Length);
Array.Clear(easylistLines, 0, easylistLines.Length);
adultLines = null;
easylistLines = null;
System.GC.Collect();
Console.WriteLine();
Console.WriteLine("Testing Parse And Store To DB Speed");
//var dbOutPath = AppDomain.CurrentDomain.BaseDirectory + "Test.db";
//var filterCollection = new FilterDbCollection(dbOutPath);
var filterCollection = new FilterDbCollection(Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "rules.db"), true, true);
var adultFileStream = File.OpenRead(adultDomainsPath);
var easylistFileStream = File.OpenRead(easylistPath);
sw.Restart();
var adultResult = filterCollection.ParseStoreRulesFromStream(adultFileStream, 1);
var easyListResult = filterCollection.ParseStoreRulesFromStream(easylistFileStream, 2);
// Ensure that we build the index AFTER we're all done our inserts.
filterCollection.FinalizeForRead();
filterCollection.InitializeBloomFilters();
sw.Stop();
Console.WriteLine("Parsed And Stored {0} filters in {1} msec, averaging {2} msec per filter.", adultResult.Item1 + easyListResult.Item1, sw.ElapsedMilliseconds, sw.ElapsedMilliseconds / (double)(adultResult.Item1 + easyListResult.Item1));
// Test a few bloom filter thingies.
TestBloomFilter(filterCollection, "pornhub.com", false);
TestBloomFilter(filterCollection, "somebogusdomain.com", false);
TestBloomFilter(filterCollection, "somebogusdomain.com", true);
TestBloomFilter(filterCollection, "disqus.com", false);
TestBloomFilter(filterCollection, "disqus.com", true);
Console.WriteLine();
Console.WriteLine("Testing Rule Lookup By Domain From DB");
int loadedFilters = 0;
sw.Restart();
for (int i = 0; i < 1000; ++i)
{
loadedFilters += filterCollection.GetFiltersForDomain().Count();
}
sw.Stop();
Console.WriteLine("Looked up and reconstructed {0} filters from DB in {1} msec, averaging {2} msec per lookup and {3} msec per filter lookup and reconstruction.", loadedFilters, sw.ElapsedMilliseconds, sw.ElapsedMilliseconds / (double)(1000), sw.ElapsedMilliseconds / (double)(loadedFilters));
Console.WriteLine();
TestFilterMatching();
Console.WriteLine("Press any key to exit...");
Console.ReadKey();
}