/// <summary>
/// Constructs a new FilterDbCollection.
/// </summary>
/// <param name="dbAbsolutePath">
/// The absolute path where the database exists or should be created.
/// </param>
/// <param name="overwrite">
/// If true, and a file exists at the supplied absolute db path, it will be deleted first.
/// Default is true.
/// </param>
/// <param name="useMemory">
/// If true, the database will be created as a purely in-memory database.
/// </param>
/// <param name="cacheOptions">
/// User defined query caching options.
/// </param>
public FilterDbCollection(string dbAbsolutePath, bool overwrite = true, bool useMemory = false, MemoryCacheOptions cacheOptions = null)
{
if (!useMemory && overwrite && File.Exists(dbAbsolutePath))
{
File.Delete(dbAbsolutePath);
}
if (cacheOptions == null)
{
cacheOptions = new MemoryCacheOptions
{
ExpirationScanFrequency = TimeSpan.FromMinutes(10)
};
}
m_cacheOptions = cacheOptions;
RecreateCache();
bool isNew = !File.Exists(dbAbsolutePath);
m_ruleParser = new AbpFormatRuleParser();
if (useMemory)
{
var version = typeof(FilterDbCollection).Assembly.GetCustomAttribute <AssemblyFileVersionAttribute>().Version;
var rnd = new Random();
var rndNum = rnd.Next();
var generatedDbName = string.Format("{0} {1} - {2}", nameof(FilterDbCollection), version, rndNum);
// "Data Source = :memory:; Cache = shared;"
var cb = new SqliteConnectionStringBuilder
{
Mode = SqliteOpenMode.Memory,
Cache = SqliteCacheMode.Shared,
DataSource = generatedDbName
};
m_connection = new SqliteConnection(cb.ToString());
}
else
{
// "Data Source={0};"
var cb = new SqliteConnectionStringBuilder
{
Mode = SqliteOpenMode.ReadWriteCreate,
Cache = SqliteCacheMode.Shared,
DataSource = dbAbsolutePath
};
m_connection = new SqliteConnection(cb.ToString());
}
//m_connection. = SQLiteConnectionFlags.UseConnectionPool | SQLiteConnectionFlags.NoConvertSettings | SQLiteConnectionFlags.NoVerifyTypeAffinity;
m_connection.Open();
ConfigureDatabase();
CreateTables();
m_globalKey = "global";
}
private static void TestFilterMatching()
{
// XXX TODO - This is the basis to begin a good benchmark, but is
// not one itself. Our speeds will be skewed by caching and such,
// so to really bench this, we need more random, randomly ordered
// "in-the-wild" kind of stuff.
var rp = new AbpFormatRuleParser();
var filter = rp.ParseAbpFormattedRule("||silly.com^stoopid^url^*1$xmlhttprequest,script,~third-party", 1) as UrlFilter;
var headers = new NameValueCollection(StringComparer.OrdinalIgnoreCase)
{
{ "X-Requested-With", "XmlHttpRequest" },
{ "Content-Type", "script" },
};
var uri = new Uri("http://silly.com/stoopid/url&=b1");
double d = 10000000;
var results = new List <bool>((int)d);
var sw2 = new Stopwatch();
Console.WriteLine("Roughly Benchmarking Filter Matching Speed");
sw2.Start();
for (int i = 0; i < d; ++i)
{
results.Add(filter.IsMatch(uri, headers));
}
sw2.Stop();
// Should be less than a microsecond.
Console.WriteLine("Filter matching loosely benchmarked at {0} microseconds per check.", ((sw2.ElapsedMilliseconds * 1000) / d));
}
/// <summary>
/// Constructs a new FilterDbCollection using an in-memory database.
/// </summary>
public FilterDbCollection()
{
var version = typeof(FilterDbCollection).Assembly.GetCustomAttribute <AssemblyFileVersionAttribute>().Version;
var rnd = new Random();
var rndNum = rnd.Next();
var generatedDbName = string.Format("{0} {1} - {2}", nameof(FilterDbCollection), version, rndNum);
// "Data Source = :memory:; Cache = shared;"
var cb = new SqliteConnectionStringBuilder();
cb.DataSource = generatedDbName;
cb.Mode = SqliteOpenMode.Memory;
cb.Cache = SqliteCacheMode.Shared;
m_connection = new SqliteConnection(cb.ToString());
//m_connection. = SQLiteConnectionFlags.UseConnectionPool | SQLiteConnectionFlags.NoConvertSettings | SQLiteConnectionFlags.NoVerifyTypeAffinity;
m_connection.Open();
ConfigureDatabase();
CreateTables();
m_globalKey = "global";
m_ruleParser = new AbpFormatRuleParser();
}
private static void Main(string[] args)
{
var parser = new AbpFormatRuleParser();
string easylistPath = AppDomain.CurrentDomain.BaseDirectory + "easylist.txt";
string adultDomainsPath = AppDomain.CurrentDomain.BaseDirectory + "adult_domains.txt";
var easylistLines = File.ReadAllLines(easylistPath);
var adultLines = File.ReadAllLines(adultDomainsPath);
var sw = new Stopwatch();
Console.WriteLine("Testing Parser Speed");
Console.WriteLine("About To Parse {0} Filters From Easylist", easylistLines.Length);
var compiledFilters = new List <Filter>(easylistLines.Length);
sw.Start();
foreach (var entry in easylistLines)
{
compiledFilters.Add(parser.ParseAbpFormattedRule(entry, 1));
}
sw.Stop();
Console.WriteLine("Parsed {0} Easylist filters in {1} msec, averaging {2} msec per filter.", compiledFilters.Count, sw.ElapsedMilliseconds, sw.ElapsedMilliseconds / (double)compiledFilters.Count);
compiledFilters = new List <Filter>(adultDomainsPath.Length);
Console.WriteLine();
Console.WriteLine("Testing Parser Speed");
Console.WriteLine("About To Parse {0} Filters From Adult Domains", adultLines.Length);
sw.Restart();
foreach (var entry in adultLines)
{
compiledFilters.Add(parser.ParseAbpFormattedRule(entry, 1));
}
sw.Stop();
Console.WriteLine("Parsed {0} Adult Domains filters in {1} msec, averaging {2} msec per filter.", compiledFilters.Count, sw.ElapsedMilliseconds, sw.ElapsedMilliseconds / (double)compiledFilters.Count);
Array.Clear(adultLines, 0, adultLines.Length);
Array.Clear(easylistLines, 0, easylistLines.Length);
adultLines = null;
easylistLines = null;
System.GC.Collect();
Console.WriteLine();
Console.WriteLine("Testing Parse And Store To DB Speed");
//var dbOutPath = AppDomain.CurrentDomain.BaseDirectory + "Test.db";
//var filterCollection = new FilterDbCollection(dbOutPath);
var filterCollection = new FilterDbCollection(Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "rules.db"), true, true);
var adultFileStream = File.OpenRead(adultDomainsPath);
var easylistFileStream = File.OpenRead(easylistPath);
sw.Restart();
var adultResult = filterCollection.ParseStoreRulesFromStream(adultFileStream, 1);
var easyListResult = filterCollection.ParseStoreRulesFromStream(easylistFileStream, 2);
// Ensure that we build the index AFTER we're all done our inserts.
filterCollection.FinalizeForRead();
filterCollection.InitializeBloomFilters();
sw.Stop();
Console.WriteLine("Parsed And Stored {0} filters in {1} msec, averaging {2} msec per filter.", adultResult.Item1 + easyListResult.Item1, sw.ElapsedMilliseconds, sw.ElapsedMilliseconds / (double)(adultResult.Item1 + easyListResult.Item1));
// Test a few bloom filter thingies.
TestBloomFilter(filterCollection, "pornhub.com", false);
TestBloomFilter(filterCollection, "somebogusdomain.com", false);
TestBloomFilter(filterCollection, "somebogusdomain.com", true);
TestBloomFilter(filterCollection, "disqus.com", false);
TestBloomFilter(filterCollection, "disqus.com", true);
Console.WriteLine();
Console.WriteLine("Testing Rule Lookup By Domain From DB");
int loadedFilters = 0;
sw.Restart();
for (int i = 0; i < 1000; ++i)
{
loadedFilters += filterCollection.GetFiltersForDomain().Count();
}
sw.Stop();
Console.WriteLine("Looked up and reconstructed {0} filters from DB in {1} msec, averaging {2} msec per lookup and {3} msec per filter lookup and reconstruction.", loadedFilters, sw.ElapsedMilliseconds, sw.ElapsedMilliseconds / (double)(1000), sw.ElapsedMilliseconds / (double)(loadedFilters));
Console.WriteLine();
TestFilterMatching();
Console.WriteLine("Press any key to exit...");
Console.ReadKey();
}
