/// <summary> /// Constructs a new FilterDbCollection. /// </summary> /// <param name="dbAbsolutePath"> /// The absolute path where the database exists or should be created. /// </param> /// <param name="overwrite"> /// If true, and a file exists at the supplied absolute db path, it will be deleted first. /// Default is true. /// </param> /// <param name="useMemory"> /// If true, the database will be created as a purely in-memory database. /// </param> /// <param name="cacheOptions"> /// User defined query caching options. /// </param> public FilterDbCollection(string dbAbsolutePath, bool overwrite = true, bool useMemory = false, MemoryCacheOptions cacheOptions = null) { if (!useMemory && overwrite && File.Exists(dbAbsolutePath)) { File.Delete(dbAbsolutePath); } if (cacheOptions == null) { cacheOptions = new MemoryCacheOptions { ExpirationScanFrequency = TimeSpan.FromMinutes(10) }; } m_cacheOptions = cacheOptions; RecreateCache(); bool isNew = !File.Exists(dbAbsolutePath); m_ruleParser = new AbpFormatRuleParser(); if (useMemory) { var version = typeof(FilterDbCollection).Assembly.GetCustomAttribute <AssemblyFileVersionAttribute>().Version; var rnd = new Random(); var rndNum = rnd.Next(); var generatedDbName = string.Format("{0} {1} - {2}", nameof(FilterDbCollection), version, rndNum); // "Data Source = :memory:; Cache = shared;" var cb = new SqliteConnectionStringBuilder { Mode = SqliteOpenMode.Memory, Cache = SqliteCacheMode.Shared, DataSource = generatedDbName }; m_connection = new SqliteConnection(cb.ToString()); } else { // "Data Source={0};" var cb = new SqliteConnectionStringBuilder { Mode = SqliteOpenMode.ReadWriteCreate, Cache = SqliteCacheMode.Shared, DataSource = dbAbsolutePath }; m_connection = new SqliteConnection(cb.ToString()); } //m_connection. = SQLiteConnectionFlags.UseConnectionPool | SQLiteConnectionFlags.NoConvertSettings | SQLiteConnectionFlags.NoVerifyTypeAffinity; m_connection.Open(); ConfigureDatabase(); CreateTables(); m_globalKey = "global"; }
private static void TestFilterMatching() { // XXX TODO - This is the basis to begin a good benchmark, but is // not one itself. Our speeds will be skewed by caching and such, // so to really bench this, we need more random, randomly ordered // "in-the-wild" kind of stuff. var rp = new AbpFormatRuleParser(); var filter = rp.ParseAbpFormattedRule("||silly.com^stoopid^url^*1$xmlhttprequest,script,~third-party", 1) as UrlFilter; var headers = new NameValueCollection(StringComparer.OrdinalIgnoreCase) { { "X-Requested-With", "XmlHttpRequest" }, { "Content-Type", "script" }, }; var uri = new Uri("http://silly.com/stoopid/url&=b1"); double d = 10000000; var results = new List <bool>((int)d); var sw2 = new Stopwatch(); Console.WriteLine("Roughly Benchmarking Filter Matching Speed"); sw2.Start(); for (int i = 0; i < d; ++i) { results.Add(filter.IsMatch(uri, headers)); } sw2.Stop(); // Should be less than a microsecond. Console.WriteLine("Filter matching loosely benchmarked at {0} microseconds per check.", ((sw2.ElapsedMilliseconds * 1000) / d)); }
/// <summary> /// Constructs a new FilterDbCollection using an in-memory database. /// </summary> public FilterDbCollection() { var version = typeof(FilterDbCollection).Assembly.GetCustomAttribute <AssemblyFileVersionAttribute>().Version; var rnd = new Random(); var rndNum = rnd.Next(); var generatedDbName = string.Format("{0} {1} - {2}", nameof(FilterDbCollection), version, rndNum); // "Data Source = :memory:; Cache = shared;" var cb = new SqliteConnectionStringBuilder(); cb.DataSource = generatedDbName; cb.Mode = SqliteOpenMode.Memory; cb.Cache = SqliteCacheMode.Shared; m_connection = new SqliteConnection(cb.ToString()); //m_connection. = SQLiteConnectionFlags.UseConnectionPool | SQLiteConnectionFlags.NoConvertSettings | SQLiteConnectionFlags.NoVerifyTypeAffinity; m_connection.Open(); ConfigureDatabase(); CreateTables(); m_globalKey = "global"; m_ruleParser = new AbpFormatRuleParser(); }
private static void Main(string[] args) { var parser = new AbpFormatRuleParser(); string easylistPath = AppDomain.CurrentDomain.BaseDirectory + "easylist.txt"; string adultDomainsPath = AppDomain.CurrentDomain.BaseDirectory + "adult_domains.txt"; var easylistLines = File.ReadAllLines(easylistPath); var adultLines = File.ReadAllLines(adultDomainsPath); var sw = new Stopwatch(); Console.WriteLine("Testing Parser Speed"); Console.WriteLine("About To Parse {0} Filters From Easylist", easylistLines.Length); var compiledFilters = new List <Filter>(easylistLines.Length); sw.Start(); foreach (var entry in easylistLines) { compiledFilters.Add(parser.ParseAbpFormattedRule(entry, 1)); } sw.Stop(); Console.WriteLine("Parsed {0} Easylist filters in {1} msec, averaging {2} msec per filter.", compiledFilters.Count, sw.ElapsedMilliseconds, sw.ElapsedMilliseconds / (double)compiledFilters.Count); compiledFilters = new List <Filter>(adultDomainsPath.Length); Console.WriteLine(); Console.WriteLine("Testing Parser Speed"); Console.WriteLine("About To Parse {0} Filters From Adult Domains", adultLines.Length); sw.Restart(); foreach (var entry in adultLines) { compiledFilters.Add(parser.ParseAbpFormattedRule(entry, 1)); } sw.Stop(); Console.WriteLine("Parsed {0} Adult Domains filters in {1} msec, averaging {2} msec per filter.", compiledFilters.Count, sw.ElapsedMilliseconds, sw.ElapsedMilliseconds / (double)compiledFilters.Count); Array.Clear(adultLines, 0, adultLines.Length); Array.Clear(easylistLines, 0, easylistLines.Length); adultLines = null; easylistLines = null; System.GC.Collect(); Console.WriteLine(); Console.WriteLine("Testing Parse And Store To DB Speed"); //var dbOutPath = AppDomain.CurrentDomain.BaseDirectory + "Test.db"; //var filterCollection = new FilterDbCollection(dbOutPath); var filterCollection = new FilterDbCollection(Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "rules.db"), true, true); var adultFileStream = File.OpenRead(adultDomainsPath); var easylistFileStream = File.OpenRead(easylistPath); sw.Restart(); var adultResult = filterCollection.ParseStoreRulesFromStream(adultFileStream, 1); var easyListResult = filterCollection.ParseStoreRulesFromStream(easylistFileStream, 2); // Ensure that we build the index AFTER we're all done our inserts. filterCollection.FinalizeForRead(); filterCollection.InitializeBloomFilters(); sw.Stop(); Console.WriteLine("Parsed And Stored {0} filters in {1} msec, averaging {2} msec per filter.", adultResult.Item1 + easyListResult.Item1, sw.ElapsedMilliseconds, sw.ElapsedMilliseconds / (double)(adultResult.Item1 + easyListResult.Item1)); // Test a few bloom filter thingies. TestBloomFilter(filterCollection, "pornhub.com", false); TestBloomFilter(filterCollection, "somebogusdomain.com", false); TestBloomFilter(filterCollection, "somebogusdomain.com", true); TestBloomFilter(filterCollection, "disqus.com", false); TestBloomFilter(filterCollection, "disqus.com", true); Console.WriteLine(); Console.WriteLine("Testing Rule Lookup By Domain From DB"); int loadedFilters = 0; sw.Restart(); for (int i = 0; i < 1000; ++i) { loadedFilters += filterCollection.GetFiltersForDomain().Count(); } sw.Stop(); Console.WriteLine("Looked up and reconstructed {0} filters from DB in {1} msec, averaging {2} msec per lookup and {3} msec per filter lookup and reconstruction.", loadedFilters, sw.ElapsedMilliseconds, sw.ElapsedMilliseconds / (double)(1000), sw.ElapsedMilliseconds / (double)(loadedFilters)); Console.WriteLine(); TestFilterMatching(); Console.WriteLine("Press any key to exit..."); Console.ReadKey(); }