/// <summary>
/// Parses the supplied list of rules and stores them in the assigned database for retrieval,
/// indexed by the rule's domain names.
/// </summary>
/// <param name="rawRuleStrings">
/// The raw filter strings.
/// </param>
/// <param name="categoryId">
/// The category ID that each of the supplied filters is deemeded to belong to.
/// </param>
/// <returns>
/// A tuple where the first item is the total number of rules successfully parsed and stored,
/// and the second item is the total number of rules that failed to be parsed and stored.
/// Failed rules are an indication of improperly formatted rules.
/// </returns>
public Tuple <int, int> ParseStoreRules(string[] rawRuleStrings, short categoryId)
{
RecreateCache();
int loaded = 0, failed = 0;
using (var transaction = m_connection.BeginTransaction())
{
using (var cmd = m_connection.CreateCommand())
{
cmd.CommandText = "INSERT INTO UrlFiltersIndex VALUES ($domain, $categoryId, $isWhitelist, $source)";
var domainParam = new SqliteParameter("$domain", DbType.String);
var categoryIdParam = new SqliteParameter("$categoryId", DbType.Int16);
var isWhitelistParam = new SqliteParameter("$isWhitelist", DbType.Boolean);
var sourceParam = new SqliteParameter("$source", DbType.String);
cmd.Parameters.Add(domainParam);
cmd.Parameters.Add(categoryIdParam);
cmd.Parameters.Add(isWhitelistParam);
cmd.Parameters.Add(sourceParam);
cmd.Prepare();
var len = rawRuleStrings.Length;
for (int i = 0; i < len; ++i)
{
rawRuleStrings[i] = rawRuleStrings[i].TrimQuick();
if (!(m_ruleParser.ParseAbpFormattedRule(rawRuleStrings[i], categoryId) is UrlFilter filter))
{
++failed;
continue;
}
++loaded;
if (filter.ApplicableDomains.Count > 0)
{
foreach (var dmn in filter.ApplicableDomains)
{
cmd.Parameters[0].Value = dmn;
cmd.Parameters[1].Value = categoryId;
cmd.Parameters[2].Value = filter.IsException;
cmd.Parameters[3].Value = rawRuleStrings[i];
cmd.ExecuteNonQuery();
}
}
else
{
cmd.Parameters[0].Value = m_globalKey;
cmd.Parameters[1].Value = categoryId;
cmd.Parameters[2].Value = filter.IsException;
cmd.Parameters[3].Value = rawRuleStrings[i];
cmd.ExecuteNonQuery();
}
}
}
transaction.Commit();
}
return(new Tuple <int, int>(loaded, failed));
}
private static void Main(string[] args)
{
var parser = new AbpFormatRuleParser();
string easylistPath = AppDomain.CurrentDomain.BaseDirectory + "easylist.txt";
string adultDomainsPath = AppDomain.CurrentDomain.BaseDirectory + "adult_domains.txt";
var easylistLines = File.ReadAllLines(easylistPath);
var adultLines = File.ReadAllLines(adultDomainsPath);
var sw = new Stopwatch();
Console.WriteLine("Testing Parser Speed");
Console.WriteLine("About To Parse {0} Filters From Easylist", easylistLines.Length);
var compiledFilters = new List <Filter>(easylistLines.Length);
sw.Start();
foreach (var entry in easylistLines)
{
compiledFilters.Add(parser.ParseAbpFormattedRule(entry, 1));
}
sw.Stop();
Console.WriteLine("Parsed {0} Easylist filters in {1} msec, averaging {2} msec per filter.", compiledFilters.Count, sw.ElapsedMilliseconds, sw.ElapsedMilliseconds / (double)compiledFilters.Count);
compiledFilters = new List <Filter>(adultDomainsPath.Length);
Console.WriteLine();
Console.WriteLine("Testing Parser Speed");
Console.WriteLine("About To Parse {0} Filters From Adult Domains", adultLines.Length);
sw.Restart();
foreach (var entry in adultLines)
{
compiledFilters.Add(parser.ParseAbpFormattedRule(entry, 1));
}
sw.Stop();
Console.WriteLine("Parsed {0} Adult Domains filters in {1} msec, averaging {2} msec per filter.", compiledFilters.Count, sw.ElapsedMilliseconds, sw.ElapsedMilliseconds / (double)compiledFilters.Count);
Array.Clear(adultLines, 0, adultLines.Length);
Array.Clear(easylistLines, 0, easylistLines.Length);
adultLines = null;
easylistLines = null;
System.GC.Collect();
Console.WriteLine();
Console.WriteLine("Testing Parse And Store To DB Speed");
//var dbOutPath = AppDomain.CurrentDomain.BaseDirectory + "Test.db";
//var filterCollection = new FilterDbCollection(dbOutPath);
var filterCollection = new FilterDbCollection(Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "rules.db"), true, true);
var adultFileStream = File.OpenRead(adultDomainsPath);
var easylistFileStream = File.OpenRead(easylistPath);
sw.Restart();
var adultResult = filterCollection.ParseStoreRulesFromStream(adultFileStream, 1);
var easyListResult = filterCollection.ParseStoreRulesFromStream(easylistFileStream, 2);
// Ensure that we build the index AFTER we're all done our inserts.
filterCollection.FinalizeForRead();
filterCollection.InitializeBloomFilters();
sw.Stop();
Console.WriteLine("Parsed And Stored {0} filters in {1} msec, averaging {2} msec per filter.", adultResult.Item1 + easyListResult.Item1, sw.ElapsedMilliseconds, sw.ElapsedMilliseconds / (double)(adultResult.Item1 + easyListResult.Item1));
// Test a few bloom filter thingies.
TestBloomFilter(filterCollection, "pornhub.com", false);
TestBloomFilter(filterCollection, "somebogusdomain.com", false);
TestBloomFilter(filterCollection, "somebogusdomain.com", true);
TestBloomFilter(filterCollection, "disqus.com", false);
TestBloomFilter(filterCollection, "disqus.com", true);
Console.WriteLine();
Console.WriteLine("Testing Rule Lookup By Domain From DB");
int loadedFilters = 0;
sw.Restart();
for (int i = 0; i < 1000; ++i)
{
loadedFilters += filterCollection.GetFiltersForDomain().Count();
}
sw.Stop();
Console.WriteLine("Looked up and reconstructed {0} filters from DB in {1} msec, averaging {2} msec per lookup and {3} msec per filter lookup and reconstruction.", loadedFilters, sw.ElapsedMilliseconds, sw.ElapsedMilliseconds / (double)(1000), sw.ElapsedMilliseconds / (double)(loadedFilters));
Console.WriteLine();
TestFilterMatching();
Console.WriteLine("Press any key to exit...");
Console.ReadKey();
}