private static bool DownloadEasyList(WebProxy proxy) { using (WebClient webClient = new WebClient()) { if (proxy != null) { webClient.Proxy = proxy; Console.WriteLine("use proxy: {0}", proxy.Address.Authority.ToString()); } Dictionary <string, string> lists = new Dictionary <string, string>(); lists.Add(ChinaListConst.EASYLIST, ChinaListConst.EASYLIST_URL); lists.Add(ChinaListConst.EASYPRIVACY, ChinaListConst.EASYPRIVACY_URL); foreach (var s in lists) { if (IsFileExist(s.Key)) { Console.WriteLine("{0} is the latest, skip over downloading.", s.Key); } else { Console.WriteLine("{0} is out of date, to start the update.", s.Key); webClient.DownloadFile(s.Value, s.Key); Console.WriteLine("update {0} completed.", s.Key); ChinaList t = new ChinaList(s.Key); if (t.Validate() != 1) { Console.WriteLine(string.Format("Download {0} error,pls try later.", s.Key)); return(false); } } } } return(true); }
/// <summary> /// validate domain by nslookup /// </summary> /// <param name="dns"></param> /// <param name="fileName"></param> /// <param name="invalidDomains"></param> public static void ValidateDomains(IPAddress dns, string fileName, string invalidDomains = "invalid_domains.txt") { if (dns == null) { dns = IPAddress.Parse("8.8.8.8"); } if (string.IsNullOrEmpty(invalidDomains)) { invalidDomains = "invalid_domains.txt"; } ChinaList cl = new ChinaList(fileName); List <string> domains = cl.GetDomains(); //List<string> urls = cl.ParseURLs(); StringBuilder results = new StringBuilder(); //StringBuilder fullResult = new StringBuilder(); List <string> whiteList = new List <string>(); whiteList.Add("ns1.dnsv2.com"); Parallel.ForEach(domains, domain => { Console.WriteLine("Querying DNS records for domain: {0}", domain); QueryResult queryResult = DNSQuery(dns, domain); Console.Write(queryResult.ToString()); //fullResult.Append(queryResult.ToString()); bool ret = false; if (queryResult.NSCount < 1) { results.Append(queryResult.ToString()); return; } foreach (var ns in queryResult.NSList) { var t = ns; if (ns.Contains("=")) { t = ParseNameServer(ns); } try { IPHostEntry ip = Dns.GetHostEntry(t); QueryResult temp = DNSQuery(ip.AddressList[0], domain); if (temp.NSCount > 0 || whiteList.Contains(t)) { ret = true; break; } else { queryResult.Error += string.Format("\n[V]: ns->{0}, Count->{1}", t, temp.NSCount); } } catch (Exception ex) { queryResult.Error += string.Format("\n[V]: ns->{0}, Error->{1}", t, ex.Message); Console.WriteLine("Validate domain: {0}, ns: {1} Error: {2}", domain, t, ex.Message); } } if (!ret) { queryResult.Error += "\n[V]: validate domian fail."; results.Append(queryResult.ToString()); } }); Save(invalidDomains, results.ToString()); }
/// <summary> /// Merge input list with part of EasyList and EasyPrivacy /// </summary> /// <param name="chinaList"></param> /// <param name="proxy"></param> /// <param name="patch"></param> /// <param name="lazyList"></param> public static void Merge(string chinaList, WebProxy proxy, bool patch, string lazyList = "adblock-lazy.txt") { if (!DownloadEasyList(proxy)) { return; } if (string.IsNullOrEmpty(lazyList)) { lazyList = "adblock-lazy.txt"; } // validate ChinaList to merge ChinaList cl = new ChinaList(chinaList); cl.Update(); if (cl.Validate() != 1) { return; } // load ChinaList content string chinaListContent = string.Empty; StringBuilder sBuilder = new StringBuilder(); using (StreamReader sr = new StreamReader(chinaList, Encoding.UTF8)) { chinaListContent = sr.ReadToEnd(); var headerIndex = chinaListContent.IndexOf(ChinaListConst.CHINALIST_LAZY_HEADER_MARK); chinaListContent = chinaListContent.Substring(headerIndex).Insert(0, ChinaListConst.CHINALIST_LAZY_HEADER); var index = chinaListContent.IndexOf(ChinaListConst.CHINALIST_END_MARK); chinaListContent = chinaListContent.Remove(index); sBuilder.Append(chinaListContent); } string easyListContent = TrimEasyList(); sBuilder.AppendLine("!-----------------------EasyList----------------------------"); sBuilder.AppendLine(easyListContent); string easyPrivacyContent = TrimEasyPrivacy(); sBuilder.AppendLine("!-----------------------EasyPrivacy----------------------------"); sBuilder.Append(easyPrivacyContent); //apply patch settings if (File.Exists(ChinaListConst.PATCH_FILE) && patch) { Console.WriteLine("use {0} to patch {1}", ChinaListConst.PATCH_FILE, lazyList); Configurations pConfig = GetConfigurations(); if (pConfig != null) { foreach (var item in pConfig.RemovedItems) { sBuilder.Replace(item + "\n", string.Empty); Console.WriteLine("remove filter {0}", item); } foreach (var item in pConfig.ModifyItems) { sBuilder.Replace(item.OldItem, item.NewItem); Console.WriteLine("replace filter {0} with {1}", item.OldItem, item.NewItem); } if (pConfig.NewItems.Count > 0) { sBuilder.AppendLine("!-----------------additional for ChinaList Lazy-------------"); } foreach (var item in pConfig.NewItems) { sBuilder.AppendLine(item); Console.WriteLine("add filter {0}", item); } // Merge ChinaList Privacy if (!string.IsNullOrEmpty(pConfig.Privacy)) { sBuilder.AppendLine("! *** adblock-privacy.txt"); var line = string.Empty; using (StreamReader sr = new StreamReader(pConfig.Privacy, Encoding.UTF8)) { while ((line = sr.ReadLine()) != null) { if (string.IsNullOrWhiteSpace(line) || line.StartsWith("!") || line.StartsWith("[")) { continue; } sBuilder.AppendLine(line); } } } } Console.WriteLine("Patch file end."); } sBuilder.AppendLine(string.Empty); sBuilder.AppendLine(ChinaListConst.CHINALIST_END_MARK); Console.WriteLine(string.Format("Merge {0}, {1} and {2}.", chinaList, ChinaListConst.EASYLIST, ChinaListConst.EASYPRIVACY)); Save(lazyList, sBuilder.ToString()); cl = new ChinaList(lazyList); cl.Update(); cl.Validate(); Console.WriteLine("End of merge and validate."); }
private static void DispatcherTask(Arguments args) { if (args.IsTrue("help") || args.IsTrue("h")) { Console.WriteLine(ChinaListConst.HELP_INFO, DateTime.Now.ToString("yyyy")); } else if (args.IsTrue("version")) { Console.WriteLine("ABPUtils version: {0}", ChinaLists.GetVersion()); } else if (args.IsTrue("ns") || args.IsTrue("nslookup")) { var domain = args.Single("d"); if (string.IsNullOrEmpty(domain)) domain = args.Single("domain"); if (string.IsNullOrEmpty(domain)) { Console.WriteLine("wrong input domain."); return; } QueryResult result = null; if (string.IsNullOrEmpty(args.Single("dns"))) result = ChinaLists.DNSQuery(null, domain); else result = ChinaLists.DNSQuery(IPAddress.Parse(args.Single("dns")), domain); if (result == null) { Console.WriteLine("Query result is null."); } else { Console.Write(result.ToString()); } } else if (args.IsTrue("v") || args.IsTrue("validate")) { var input = args.Single("i"); if (string.IsNullOrEmpty(input)) input = args.Single("input"); if (string.IsNullOrEmpty(input)) { Console.WriteLine("wrong input file."); return; } var chinaList = new ChinaList(input); chinaList.Validate(); } else if (args.IsTrue("u") || args.IsTrue("update")) { var input = args.Single("i"); if (string.IsNullOrEmpty(input)) input = args.Single("input"); if (string.IsNullOrEmpty(input)) { Console.WriteLine("wrong input file."); return; } var chinaList = new ChinaList(input); chinaList.Update(); chinaList.Validate(); } else if (args.IsTrue("c") || args.IsTrue("check")) { var input = args.Single("i"); if (string.IsNullOrEmpty(input)) input = args.Single("input"); if (string.IsNullOrEmpty(input)) { Console.WriteLine("wrong input file."); return; } var output = args.Single("o"); if (string.IsNullOrEmpty(output)) output = args.Single("output"); var dns = args.Single("dns"); if (string.IsNullOrEmpty(dns)) { ChinaLists.ValidateDomains(null, input, output); } else { ChinaLists.ValidateDomains(IPAddress.Parse(args.Single("dns")), input, output); } } else if (args.IsTrue("m") || args.IsTrue("merge")) { var input = string.Empty; input = args.Single("i"); if (string.IsNullOrEmpty(input)) input = args.Single("input"); if (string.IsNullOrEmpty(input)) { Console.WriteLine("wrong input file."); return; } WebProxy proxy = null; var p = args.Single("p"); if (string.IsNullOrEmpty(p)) p = args.Single("proxy"); if (!string.IsNullOrEmpty(p)) { var temp = p.Split(':'); proxy = new WebProxy(temp[0], int.Parse(temp[1])); proxy.BypassProxyOnLocal = true; } var output = args.Single("o"); if (string.IsNullOrEmpty(output)) output = args.Single("output"); ChinaLists.Merge(input, proxy, args.IsTrue("patch"), output); } else if (args.IsTrue("conf")) { var input = string.Empty; input = args.Single("i"); if (string.IsNullOrEmpty(input)) input = args.Single("input"); if (string.IsNullOrEmpty(input)) { Console.WriteLine("wrong input file."); return; } ChinaLists.CleanConfigurations(input, null); Console.WriteLine("Clean configuration file successful."); } else { Console.WriteLine("Wrong args"); } }
private static bool DownloadEasyList(WebProxy proxy) { using (WebClient webClient = new WebClient()) { if (proxy != null) { webClient.Proxy = proxy; Console.WriteLine("use proxy: {0}", proxy.Address.Authority.ToString()); } Dictionary<string, string> lists = new Dictionary<string, string>(); lists.Add(ChinaListConst.EASYLIST, ChinaListConst.EASYLIST_URL); lists.Add(ChinaListConst.EASYPRIVACY, ChinaListConst.EASYPRIVACY_URL); foreach (var s in lists) { if (IsFileExist(s.Key)) { Console.WriteLine("{0} is the latest, skip over downloading.", s.Key); } else { Console.WriteLine("{0} is out of date, to start the update.", s.Key); webClient.DownloadFile(s.Value, s.Key); Console.WriteLine("update {0} completed.", s.Key); ChinaList t = new ChinaList(s.Key); if (t.Validate() != 1) { Console.WriteLine(string.Format("Download {0} error,pls try later.", s.Key)); return false; } } } } return true; }
/// <summary> /// validate domain by nslookup /// </summary> /// <param name="dns"></param> /// <param name="fileName"></param> /// <param name="invalidDomains"></param> public static void ValidateDomains(IPAddress dns, string fileName, string invalidDomains = "invalid_domains.txt") { if (dns == null) dns = IPAddress.Parse("8.8.8.8"); if (string.IsNullOrEmpty(invalidDomains)) invalidDomains = "invalid_domains.txt"; ChinaList cl = new ChinaList(fileName); List<string> domains = cl.GetDomains(); //List<string> urls = cl.ParseURLs(); StringBuilder results = new StringBuilder(); //StringBuilder fullResult = new StringBuilder(); List<string> whiteList = new List<string>(); whiteList.Add("ns1.dnsv2.com"); Parallel.ForEach(domains, domain => { Console.WriteLine("Querying DNS records for domain: {0}", domain); QueryResult queryResult = DNSQuery(dns, domain); Console.Write(queryResult.ToString()); //fullResult.Append(queryResult.ToString()); bool ret = false; if (queryResult.NSCount < 1) { results.Append(queryResult.ToString()); return; } foreach (var ns in queryResult.NSList) { var t = ns; if (ns.Contains("=")) t = ParseNameServer(ns); try { IPHostEntry ip = Dns.GetHostEntry(t); QueryResult temp = DNSQuery(ip.AddressList[0], domain); if (temp.NSCount > 0 || whiteList.Contains(t)) { ret = true; break; } else { queryResult.Error += string.Format("\n[V]: ns->{0}, Count->{1}", t, temp.NSCount); } } catch (Exception ex) { queryResult.Error += string.Format("\n[V]: ns->{0}, Error->{1}", t, ex.Message); Console.WriteLine("Validate domain: {0}, ns: {1} Error: {2}", domain, t, ex.Message); } } if (!ret) { queryResult.Error += "\n[V]: validate domian fail."; results.Append(queryResult.ToString()); } }); Save(invalidDomains, results.ToString()); }
/// <summary> /// Merge input list with part of EasyList and EasyPrivacy /// </summary> /// <param name="chinaList"></param> /// <param name="proxy"></param> /// <param name="patch"></param> /// <param name="lazyList"></param> public static void Merge(string chinaList, WebProxy proxy, bool patch, string lazyList = "adblock-lazy.txt") { if (!DownloadEasyList(proxy)) return; if (string.IsNullOrEmpty(lazyList)) lazyList = "adblock-lazy.txt"; // validate ChinaList to merge ChinaList cl = new ChinaList(chinaList); cl.Update(); if (cl.Validate() != 1) return; // load ChinaList content string chinaListContent = string.Empty; StringBuilder sBuilder = new StringBuilder(); using (StreamReader sr = new StreamReader(chinaList, Encoding.UTF8)) { chinaListContent = sr.ReadToEnd(); var headerIndex = chinaListContent.IndexOf(ChinaListConst.CHINALIST_LAZY_HEADER_MARK); chinaListContent = chinaListContent.Substring(headerIndex).Insert(0, ChinaListConst.CHINALIST_LAZY_HEADER); var index = chinaListContent.IndexOf(ChinaListConst.CHINALIST_END_MARK); chinaListContent = chinaListContent.Remove(index); sBuilder.Append(chinaListContent); } string easyListContent = TrimEasyList(); sBuilder.AppendLine("!-----------------------EasyList----------------------------"); sBuilder.AppendLine(easyListContent); string easyPrivacyContent = TrimEasyPrivacy(); sBuilder.AppendLine("!-----------------------EasyPrivacy----------------------------"); sBuilder.Append(easyPrivacyContent); //apply patch settings if (File.Exists(ChinaListConst.PATCH_FILE) && patch) { Console.WriteLine("use {0} to patch {1}", ChinaListConst.PATCH_FILE, lazyList); Configurations pConfig = GetConfigurations(); if (pConfig != null) { foreach (var item in pConfig.RemovedItems) { sBuilder.Replace(item + "\n", string.Empty); Console.WriteLine("remove filter {0}", item); } foreach (var item in pConfig.ModifyItems) { sBuilder.Replace(item.OldItem, item.NewItem); Console.WriteLine("replace filter {0} with {1}", item.OldItem, item.NewItem); } if (pConfig.NewItems.Count > 0) sBuilder.AppendLine("!-----------------additional for ChinaList Lazy-------------"); foreach (var item in pConfig.NewItems) { sBuilder.AppendLine(item); Console.WriteLine("add filter {0}", item); } // Merge ChinaList Privacy if (!string.IsNullOrEmpty(pConfig.Privacy)) { sBuilder.AppendLine("! *** adblock-privacy.txt"); var line = string.Empty; using (StreamReader sr = new StreamReader(pConfig.Privacy, Encoding.UTF8)) { while ((line = sr.ReadLine()) != null) { if (string.IsNullOrWhiteSpace(line) || line.StartsWith("!") || line.StartsWith("[")) continue; sBuilder.AppendLine(line); } } } } Console.WriteLine("Patch file end."); } sBuilder.AppendLine(string.Empty); sBuilder.AppendLine(ChinaListConst.CHINALIST_END_MARK); Console.WriteLine(string.Format("Merge {0}, {1} and {2}.", chinaList, ChinaListConst.EASYLIST, ChinaListConst.EASYPRIVACY)); Save(lazyList, sBuilder.ToString()); cl = new ChinaList(lazyList); cl.Update(); cl.Validate(); Console.WriteLine("End of merge and validate."); }
private static void DispatcherTask(Arguments args) { if (args.IsTrue("help") || args.IsTrue("h")) { Console.WriteLine(ChinaListConst.HELP_INFO, DateTime.Now.ToString("yyyy")); } else if (args.IsTrue("version")) { Console.WriteLine("ABPUtils version: {0}", ChinaLists.GetVersion()); } else if (args.IsTrue("ns") || args.IsTrue("nslookup")) { var domain = args.Single("d"); if (string.IsNullOrEmpty(domain)) { domain = args.Single("domain"); } if (string.IsNullOrEmpty(domain)) { Console.WriteLine("wrong input domain."); return; } QueryResult result = null; if (string.IsNullOrEmpty(args.Single("dns"))) { result = ChinaLists.DNSQuery(null, domain); } else { result = ChinaLists.DNSQuery(IPAddress.Parse(args.Single("dns")), domain); } if (result == null) { Console.WriteLine("Query result is null."); } else { Console.Write(result.ToString()); } } else if (args.IsTrue("v") || args.IsTrue("validate")) { var input = args.Single("i"); if (string.IsNullOrEmpty(input)) { input = args.Single("input"); } if (string.IsNullOrEmpty(input)) { Console.WriteLine("wrong input file."); return; } var chinaList = new ChinaList(input); chinaList.Validate(); } else if (args.IsTrue("u") || args.IsTrue("update")) { var input = args.Single("i"); if (string.IsNullOrEmpty(input)) { input = args.Single("input"); } if (string.IsNullOrEmpty(input)) { Console.WriteLine("wrong input file."); return; } var chinaList = new ChinaList(input); chinaList.Update(); chinaList.Validate(); } else if (args.IsTrue("c") || args.IsTrue("check")) { var input = args.Single("i"); if (string.IsNullOrEmpty(input)) { input = args.Single("input"); } if (string.IsNullOrEmpty(input)) { Console.WriteLine("wrong input file."); return; } var output = args.Single("o"); if (string.IsNullOrEmpty(output)) { output = args.Single("output"); } var dns = args.Single("dns"); if (string.IsNullOrEmpty(dns)) { ChinaLists.ValidateDomains(null, input, output); } else { ChinaLists.ValidateDomains(IPAddress.Parse(args.Single("dns")), input, output); } } else if (args.IsTrue("m") || args.IsTrue("merge")) { var input = string.Empty; input = args.Single("i"); if (string.IsNullOrEmpty(input)) { input = args.Single("input"); } if (string.IsNullOrEmpty(input)) { Console.WriteLine("wrong input file."); return; } WebProxy proxy = null; var p = args.Single("p"); if (string.IsNullOrEmpty(p)) { p = args.Single("proxy"); } if (!string.IsNullOrEmpty(p)) { var temp = p.Split(':'); proxy = new WebProxy(temp[0], int.Parse(temp[1])); proxy.BypassProxyOnLocal = true; } var output = args.Single("o"); if (string.IsNullOrEmpty(output)) { output = args.Single("output"); } ChinaLists.Merge(input, proxy, args.IsTrue("patch"), output); } else if (args.IsTrue("conf")) { var input = string.Empty; input = args.Single("i"); if (string.IsNullOrEmpty(input)) { input = args.Single("input"); } if (string.IsNullOrEmpty(input)) { Console.WriteLine("wrong input file."); return; } ChinaLists.CleanConfigurations(input, null); Console.WriteLine("Clean configuration file successful."); } else { Console.WriteLine("Wrong args"); } }