private static bool DownloadEasyList(WebProxy proxy)
{
using (WebClient webClient = new WebClient())
{
if (proxy != null)
{
webClient.Proxy = proxy;
Console.WriteLine("use proxy: {0}", proxy.Address.Authority.ToString());
}
Dictionary <string, string> lists = new Dictionary <string, string>();
lists.Add(ChinaListConst.EASYLIST, ChinaListConst.EASYLIST_URL);
lists.Add(ChinaListConst.EASYPRIVACY, ChinaListConst.EASYPRIVACY_URL);
foreach (var s in lists)
{
if (IsFileExist(s.Key))
{
Console.WriteLine("{0} is the latest, skip over downloading.", s.Key);
}
else
{
Console.WriteLine("{0} is out of date, to start the update.", s.Key);
webClient.DownloadFile(s.Value, s.Key);
Console.WriteLine("update {0} completed.", s.Key);
ChinaList t = new ChinaList(s.Key);
if (t.Validate() != 1)
{
Console.WriteLine(string.Format("Download {0} error,pls try later.", s.Key));
return(false);
}
}
}
}
return(true);
}
/// <summary>
/// validate domain by nslookup
/// </summary>
/// <param name="dns"></param>
/// <param name="fileName"></param>
/// <param name="invalidDomains"></param>
public static void ValidateDomains(IPAddress dns, string fileName, string invalidDomains = "invalid_domains.txt")
{
if (dns == null)
{
dns = IPAddress.Parse("8.8.8.8");
}
if (string.IsNullOrEmpty(invalidDomains))
{
invalidDomains = "invalid_domains.txt";
}
ChinaList cl = new ChinaList(fileName);
List <string> domains = cl.GetDomains();
//List<string> urls = cl.ParseURLs();
StringBuilder results = new StringBuilder();
//StringBuilder fullResult = new StringBuilder();
List <string> whiteList = new List <string>();
whiteList.Add("ns1.dnsv2.com");
Parallel.ForEach(domains, domain =>
{
Console.WriteLine("Querying DNS records for domain: {0}", domain);
QueryResult queryResult = DNSQuery(dns, domain);
Console.Write(queryResult.ToString());
//fullResult.Append(queryResult.ToString());
bool ret = false;
if (queryResult.NSCount < 1)
{
results.Append(queryResult.ToString());
return;
}
foreach (var ns in queryResult.NSList)
{
var t = ns;
if (ns.Contains("="))
{
t = ParseNameServer(ns);
}
try
{
IPHostEntry ip = Dns.GetHostEntry(t);
QueryResult temp = DNSQuery(ip.AddressList[0], domain);
if (temp.NSCount > 0 || whiteList.Contains(t))
{
ret = true;
break;
}
else
{
queryResult.Error += string.Format("\n[V]: ns->{0}, Count->{1}", t, temp.NSCount);
}
}
catch (Exception ex)
{
queryResult.Error += string.Format("\n[V]: ns->{0}, Error->{1}", t, ex.Message);
Console.WriteLine("Validate domain: {0}, ns: {1} Error: {2}", domain, t, ex.Message);
}
}
if (!ret)
{
queryResult.Error += "\n[V]: validate domian fail.";
results.Append(queryResult.ToString());
}
});
Save(invalidDomains, results.ToString());
}
/// <summary>
/// Merge input list with part of EasyList and EasyPrivacy
/// </summary>
/// <param name="chinaList"></param>
/// <param name="proxy"></param>
/// <param name="patch"></param>
/// <param name="lazyList"></param>
public static void Merge(string chinaList, WebProxy proxy, bool patch, string lazyList = "adblock-lazy.txt")
{
if (!DownloadEasyList(proxy))
{
return;
}
if (string.IsNullOrEmpty(lazyList))
{
lazyList = "adblock-lazy.txt";
}
// validate ChinaList to merge
ChinaList cl = new ChinaList(chinaList);
cl.Update();
if (cl.Validate() != 1)
{
return;
}
// load ChinaList content
string chinaListContent = string.Empty;
StringBuilder sBuilder = new StringBuilder();
using (StreamReader sr = new StreamReader(chinaList, Encoding.UTF8))
{
chinaListContent = sr.ReadToEnd();
var headerIndex = chinaListContent.IndexOf(ChinaListConst.CHINALIST_LAZY_HEADER_MARK);
chinaListContent = chinaListContent.Substring(headerIndex).Insert(0, ChinaListConst.CHINALIST_LAZY_HEADER);
var index = chinaListContent.IndexOf(ChinaListConst.CHINALIST_END_MARK);
chinaListContent = chinaListContent.Remove(index);
sBuilder.Append(chinaListContent);
}
string easyListContent = TrimEasyList();
sBuilder.AppendLine("!-----------------------EasyList----------------------------");
sBuilder.AppendLine(easyListContent);
string easyPrivacyContent = TrimEasyPrivacy();
sBuilder.AppendLine("!-----------------------EasyPrivacy----------------------------");
sBuilder.Append(easyPrivacyContent);
//apply patch settings
if (File.Exists(ChinaListConst.PATCH_FILE) && patch)
{
Console.WriteLine("use {0} to patch {1}", ChinaListConst.PATCH_FILE, lazyList);
Configurations pConfig = GetConfigurations();
if (pConfig != null)
{
foreach (var item in pConfig.RemovedItems)
{
sBuilder.Replace(item + "\n", string.Empty);
Console.WriteLine("remove filter {0}", item);
}
foreach (var item in pConfig.ModifyItems)
{
sBuilder.Replace(item.OldItem, item.NewItem);
Console.WriteLine("replace filter {0} with {1}", item.OldItem, item.NewItem);
}
if (pConfig.NewItems.Count > 0)
{
sBuilder.AppendLine("!-----------------additional for ChinaList Lazy-------------");
}
foreach (var item in pConfig.NewItems)
{
sBuilder.AppendLine(item);
Console.WriteLine("add filter {0}", item);
}
// Merge ChinaList Privacy
if (!string.IsNullOrEmpty(pConfig.Privacy))
{
sBuilder.AppendLine("! *** adblock-privacy.txt");
var line = string.Empty;
using (StreamReader sr = new StreamReader(pConfig.Privacy, Encoding.UTF8))
{
while ((line = sr.ReadLine()) != null)
{
if (string.IsNullOrWhiteSpace(line) || line.StartsWith("!") || line.StartsWith("["))
{
continue;
}
sBuilder.AppendLine(line);
}
}
}
}
Console.WriteLine("Patch file end.");
}
sBuilder.AppendLine(string.Empty);
sBuilder.AppendLine(ChinaListConst.CHINALIST_END_MARK);
Console.WriteLine(string.Format("Merge {0}, {1} and {2}.", chinaList, ChinaListConst.EASYLIST, ChinaListConst.EASYPRIVACY));
Save(lazyList, sBuilder.ToString());
cl = new ChinaList(lazyList);
cl.Update();
cl.Validate();
Console.WriteLine("End of merge and validate.");
}
private static void DispatcherTask(Arguments args)
{
if (args.IsTrue("help") || args.IsTrue("h"))
{
Console.WriteLine(ChinaListConst.HELP_INFO, DateTime.Now.ToString("yyyy"));
}
else if (args.IsTrue("version"))
{
Console.WriteLine("ABPUtils version: {0}", ChinaLists.GetVersion());
}
else if (args.IsTrue("ns") || args.IsTrue("nslookup"))
{
var domain = args.Single("d");
if (string.IsNullOrEmpty(domain))
domain = args.Single("domain");
if (string.IsNullOrEmpty(domain))
{
Console.WriteLine("wrong input domain.");
return;
}
QueryResult result = null;
if (string.IsNullOrEmpty(args.Single("dns")))
result = ChinaLists.DNSQuery(null, domain);
else
result = ChinaLists.DNSQuery(IPAddress.Parse(args.Single("dns")), domain);
if (result == null)
{
Console.WriteLine("Query result is null.");
}
else
{
Console.Write(result.ToString());
}
}
else if (args.IsTrue("v") || args.IsTrue("validate"))
{
var input = args.Single("i");
if (string.IsNullOrEmpty(input))
input = args.Single("input");
if (string.IsNullOrEmpty(input))
{
Console.WriteLine("wrong input file.");
return;
}
var chinaList = new ChinaList(input);
chinaList.Validate();
}
else if (args.IsTrue("u") || args.IsTrue("update"))
{
var input = args.Single("i");
if (string.IsNullOrEmpty(input))
input = args.Single("input");
if (string.IsNullOrEmpty(input))
{
Console.WriteLine("wrong input file.");
return;
}
var chinaList = new ChinaList(input);
chinaList.Update();
chinaList.Validate();
}
else if (args.IsTrue("c") || args.IsTrue("check"))
{
var input = args.Single("i");
if (string.IsNullOrEmpty(input))
input = args.Single("input");
if (string.IsNullOrEmpty(input))
{
Console.WriteLine("wrong input file.");
return;
}
var output = args.Single("o");
if (string.IsNullOrEmpty(output))
output = args.Single("output");
var dns = args.Single("dns");
if (string.IsNullOrEmpty(dns))
{
ChinaLists.ValidateDomains(null, input, output);
}
else
{
ChinaLists.ValidateDomains(IPAddress.Parse(args.Single("dns")), input, output);
}
}
else if (args.IsTrue("m") || args.IsTrue("merge"))
{
var input = string.Empty;
input = args.Single("i");
if (string.IsNullOrEmpty(input))
input = args.Single("input");
if (string.IsNullOrEmpty(input))
{
Console.WriteLine("wrong input file.");
return;
}
WebProxy proxy = null;
var p = args.Single("p");
if (string.IsNullOrEmpty(p))
p = args.Single("proxy");
if (!string.IsNullOrEmpty(p))
{
var temp = p.Split(':');
proxy = new WebProxy(temp[0], int.Parse(temp[1]));
proxy.BypassProxyOnLocal = true;
}
var output = args.Single("o");
if (string.IsNullOrEmpty(output))
output = args.Single("output");
ChinaLists.Merge(input, proxy, args.IsTrue("patch"), output);
}
else if (args.IsTrue("conf"))
{
var input = string.Empty;
input = args.Single("i");
if (string.IsNullOrEmpty(input))
input = args.Single("input");
if (string.IsNullOrEmpty(input))
{
Console.WriteLine("wrong input file.");
return;
}
ChinaLists.CleanConfigurations(input, null);
Console.WriteLine("Clean configuration file successful.");
}
else
{
Console.WriteLine("Wrong args");
}
}
private static bool DownloadEasyList(WebProxy proxy)
{
using (WebClient webClient = new WebClient())
{
if (proxy != null)
{
webClient.Proxy = proxy;
Console.WriteLine("use proxy: {0}", proxy.Address.Authority.ToString());
}
Dictionary<string, string> lists = new Dictionary<string, string>();
lists.Add(ChinaListConst.EASYLIST, ChinaListConst.EASYLIST_URL);
lists.Add(ChinaListConst.EASYPRIVACY, ChinaListConst.EASYPRIVACY_URL);
foreach (var s in lists)
{
if (IsFileExist(s.Key))
{
Console.WriteLine("{0} is the latest, skip over downloading.", s.Key);
}
else
{
Console.WriteLine("{0} is out of date, to start the update.", s.Key);
webClient.DownloadFile(s.Value, s.Key);
Console.WriteLine("update {0} completed.", s.Key);
ChinaList t = new ChinaList(s.Key);
if (t.Validate() != 1)
{
Console.WriteLine(string.Format("Download {0} error,pls try later.", s.Key));
return false;
}
}
}
}
return true;
}
/// <summary>
/// validate domain by nslookup
/// </summary>
/// <param name="dns"></param>
/// <param name="fileName"></param>
/// <param name="invalidDomains"></param>
public static void ValidateDomains(IPAddress dns, string fileName, string invalidDomains = "invalid_domains.txt")
{
if (dns == null)
dns = IPAddress.Parse("8.8.8.8");
if (string.IsNullOrEmpty(invalidDomains))
invalidDomains = "invalid_domains.txt";
ChinaList cl = new ChinaList(fileName);
List<string> domains = cl.GetDomains();
//List<string> urls = cl.ParseURLs();
StringBuilder results = new StringBuilder();
//StringBuilder fullResult = new StringBuilder();
List<string> whiteList = new List<string>();
whiteList.Add("ns1.dnsv2.com");
Parallel.ForEach(domains, domain =>
{
Console.WriteLine("Querying DNS records for domain: {0}", domain);
QueryResult queryResult = DNSQuery(dns, domain);
Console.Write(queryResult.ToString());
//fullResult.Append(queryResult.ToString());
bool ret = false;
if (queryResult.NSCount < 1)
{
results.Append(queryResult.ToString());
return;
}
foreach (var ns in queryResult.NSList)
{
var t = ns;
if (ns.Contains("="))
t = ParseNameServer(ns);
try
{
IPHostEntry ip = Dns.GetHostEntry(t);
QueryResult temp = DNSQuery(ip.AddressList[0], domain);
if (temp.NSCount > 0 || whiteList.Contains(t))
{
ret = true;
break;
}
else
{
queryResult.Error += string.Format("\n[V]: ns->{0}, Count->{1}", t, temp.NSCount);
}
}
catch (Exception ex)
{
queryResult.Error += string.Format("\n[V]: ns->{0}, Error->{1}", t, ex.Message);
Console.WriteLine("Validate domain: {0}, ns: {1} Error: {2}", domain, t, ex.Message);
}
}
if (!ret)
{
queryResult.Error += "\n[V]: validate domian fail.";
results.Append(queryResult.ToString());
}
});
Save(invalidDomains, results.ToString());
}
/// <summary>
/// Merge input list with part of EasyList and EasyPrivacy
/// </summary>
/// <param name="chinaList"></param>
/// <param name="proxy"></param>
/// <param name="patch"></param>
/// <param name="lazyList"></param>
public static void Merge(string chinaList, WebProxy proxy, bool patch, string lazyList = "adblock-lazy.txt")
{
if (!DownloadEasyList(proxy))
return;
if (string.IsNullOrEmpty(lazyList))
lazyList = "adblock-lazy.txt";
// validate ChinaList to merge
ChinaList cl = new ChinaList(chinaList);
cl.Update();
if (cl.Validate() != 1)
return;
// load ChinaList content
string chinaListContent = string.Empty;
StringBuilder sBuilder = new StringBuilder();
using (StreamReader sr = new StreamReader(chinaList, Encoding.UTF8))
{
chinaListContent = sr.ReadToEnd();
var headerIndex = chinaListContent.IndexOf(ChinaListConst.CHINALIST_LAZY_HEADER_MARK);
chinaListContent = chinaListContent.Substring(headerIndex).Insert(0, ChinaListConst.CHINALIST_LAZY_HEADER);
var index = chinaListContent.IndexOf(ChinaListConst.CHINALIST_END_MARK);
chinaListContent = chinaListContent.Remove(index);
sBuilder.Append(chinaListContent);
}
string easyListContent = TrimEasyList();
sBuilder.AppendLine("!-----------------------EasyList----------------------------");
sBuilder.AppendLine(easyListContent);
string easyPrivacyContent = TrimEasyPrivacy();
sBuilder.AppendLine("!-----------------------EasyPrivacy----------------------------");
sBuilder.Append(easyPrivacyContent);
//apply patch settings
if (File.Exists(ChinaListConst.PATCH_FILE) && patch)
{
Console.WriteLine("use {0} to patch {1}", ChinaListConst.PATCH_FILE, lazyList);
Configurations pConfig = GetConfigurations();
if (pConfig != null)
{
foreach (var item in pConfig.RemovedItems)
{
sBuilder.Replace(item + "\n", string.Empty);
Console.WriteLine("remove filter {0}", item);
}
foreach (var item in pConfig.ModifyItems)
{
sBuilder.Replace(item.OldItem, item.NewItem);
Console.WriteLine("replace filter {0} with {1}", item.OldItem, item.NewItem);
}
if (pConfig.NewItems.Count > 0)
sBuilder.AppendLine("!-----------------additional for ChinaList Lazy-------------");
foreach (var item in pConfig.NewItems)
{
sBuilder.AppendLine(item);
Console.WriteLine("add filter {0}", item);
}
// Merge ChinaList Privacy
if (!string.IsNullOrEmpty(pConfig.Privacy))
{
sBuilder.AppendLine("! *** adblock-privacy.txt");
var line = string.Empty;
using (StreamReader sr = new StreamReader(pConfig.Privacy, Encoding.UTF8))
{
while ((line = sr.ReadLine()) != null)
{
if (string.IsNullOrWhiteSpace(line) || line.StartsWith("!") || line.StartsWith("["))
continue;
sBuilder.AppendLine(line);
}
}
}
}
Console.WriteLine("Patch file end.");
}
sBuilder.AppendLine(string.Empty);
sBuilder.AppendLine(ChinaListConst.CHINALIST_END_MARK);
Console.WriteLine(string.Format("Merge {0}, {1} and {2}.", chinaList, ChinaListConst.EASYLIST, ChinaListConst.EASYPRIVACY));
Save(lazyList, sBuilder.ToString());
cl = new ChinaList(lazyList);
cl.Update();
cl.Validate();
Console.WriteLine("End of merge and validate.");
}
private static void DispatcherTask(Arguments args)
{
if (args.IsTrue("help") || args.IsTrue("h"))
{
Console.WriteLine(ChinaListConst.HELP_INFO, DateTime.Now.ToString("yyyy"));
}
else if (args.IsTrue("version"))
{
Console.WriteLine("ABPUtils version: {0}", ChinaLists.GetVersion());
}
else if (args.IsTrue("ns") || args.IsTrue("nslookup"))
{
var domain = args.Single("d");
if (string.IsNullOrEmpty(domain))
{
domain = args.Single("domain");
}
if (string.IsNullOrEmpty(domain))
{
Console.WriteLine("wrong input domain.");
return;
}
QueryResult result = null;
if (string.IsNullOrEmpty(args.Single("dns")))
{
result = ChinaLists.DNSQuery(null, domain);
}
else
{
result = ChinaLists.DNSQuery(IPAddress.Parse(args.Single("dns")), domain);
}
if (result == null)
{
Console.WriteLine("Query result is null.");
}
else
{
Console.Write(result.ToString());
}
}
else if (args.IsTrue("v") || args.IsTrue("validate"))
{
var input = args.Single("i");
if (string.IsNullOrEmpty(input))
{
input = args.Single("input");
}
if (string.IsNullOrEmpty(input))
{
Console.WriteLine("wrong input file.");
return;
}
var chinaList = new ChinaList(input);
chinaList.Validate();
}
else if (args.IsTrue("u") || args.IsTrue("update"))
{
var input = args.Single("i");
if (string.IsNullOrEmpty(input))
{
input = args.Single("input");
}
if (string.IsNullOrEmpty(input))
{
Console.WriteLine("wrong input file.");
return;
}
var chinaList = new ChinaList(input);
chinaList.Update();
chinaList.Validate();
}
else if (args.IsTrue("c") || args.IsTrue("check"))
{
var input = args.Single("i");
if (string.IsNullOrEmpty(input))
{
input = args.Single("input");
}
if (string.IsNullOrEmpty(input))
{
Console.WriteLine("wrong input file.");
return;
}
var output = args.Single("o");
if (string.IsNullOrEmpty(output))
{
output = args.Single("output");
}
var dns = args.Single("dns");
if (string.IsNullOrEmpty(dns))
{
ChinaLists.ValidateDomains(null, input, output);
}
else
{
ChinaLists.ValidateDomains(IPAddress.Parse(args.Single("dns")), input, output);
}
}
else if (args.IsTrue("m") || args.IsTrue("merge"))
{
var input = string.Empty;
input = args.Single("i");
if (string.IsNullOrEmpty(input))
{
input = args.Single("input");
}
if (string.IsNullOrEmpty(input))
{
Console.WriteLine("wrong input file.");
return;
}
WebProxy proxy = null;
var p = args.Single("p");
if (string.IsNullOrEmpty(p))
{
p = args.Single("proxy");
}
if (!string.IsNullOrEmpty(p))
{
var temp = p.Split(':');
proxy = new WebProxy(temp[0], int.Parse(temp[1]));
proxy.BypassProxyOnLocal = true;
}
var output = args.Single("o");
if (string.IsNullOrEmpty(output))
{
output = args.Single("output");
}
ChinaLists.Merge(input, proxy, args.IsTrue("patch"), output);
}
else if (args.IsTrue("conf"))
{
var input = string.Empty;
input = args.Single("i");
if (string.IsNullOrEmpty(input))
{
input = args.Single("input");
}
if (string.IsNullOrEmpty(input))
{
Console.WriteLine("wrong input file.");
return;
}
ChinaLists.CleanConfigurations(input, null);
Console.WriteLine("Clean configuration file successful.");
}
else
{
Console.WriteLine("Wrong args");
}
}
