static async Task Main(string[] args)
{
FileHelperClass fileHelper = new FileHelperClass();
UrlParserClass urlParser = new UrlParserClass();
FilterHelperClass filterHelper = new FilterHelperClass();
Console.WriteLine("Please drag&drop your query-list.", Color.LightPink);
filterHelper.addFilters("yahoo.com");
filterHelper.addFilters("google.com");
filterHelper.addFilters("bing.com");
filterHelper.addFilters("bingj.com");
filterHelper.addFilters("live.com");
filterHelper.addFilters("w3.org");
filterHelper.addFilters("microsofttranslator.com");
filterHelper.addFilters("wikipedia.org");
filterHelper.addFilters("twitter.com");
filterHelper.addFilters("youtube.com");
filterHelper.addFilters("facebook.com");
filterHelper.addFilters("instagram.com");
filterHelper.addFilters("microsoft.com");
filterHelper.addFilters("giga.de");
filterHelper.addFilters("msn.com");
filterHelper.addFilters("outlook.com");
filterHelper.addFilters("creativecommons.org");
filterHelper.addFilters("trustscam.nl");
filterHelper.addFilters("aol.de");
filterHelper.addFilters("yandex.com");
filterHelper.addFilters("verbraucherschutz.de");
filterHelper.addFilters("whois.com");
filterHelper.addFilters("bingparachute.com");
filterHelper.addFilters("duckduckgo.com");
string[] engines = { "https://www.bing.com/search?q=" };
string toReplace = '"'.ToString();
string input = Console.ReadLine().Replace(toReplace, string.Empty);
Console.WriteLine(input);
Console.Clear();
string[] querys = fileHelper.readLinesFromFile(input);
List <string> foundUrls = new List <string>();
foreach (string engine in engines)
{
Console.WriteLine("[Current Engine]: " + engine, Color.LightYellow);
foreach (string query in querys)
{
try
{
string url = engine + query;
Console.WriteLine("[Current URL]: " + url, Color.LightGreen);
string content = await urlParser.GetUrlContent(url);
string[] tmpFoundUrls = urlParser.SearchForUrls(content);
foreach (string tmpFoundUrl in tmpFoundUrls)
{
if (filterHelper.isFiltered(tmpFoundUrl))
{
foundUrls.Add(tmpFoundUrl);
Console.WriteLine("[New URL]: " + tmpFoundUrl, Color.LightPink);
}
}
}
catch (Exception e)
{
Console.WriteLine("[ERROR]: " + e.Message, Color.Red);
}
}
}
List <string> distinctUrls = foundUrls.Distinct().ToList();
Console.Clear();
Console.WriteLine("[Success!]: URL's we're scraped successfully!");
Console.WriteLine("[Information]: New URL's found: " + foundUrls.Count(), Color.LightBlue);
Console.WriteLine("[Information]: Distinct URL's found: " + distinctUrls.Count(), Color.LightPink);
fileHelper.saveToFile(distinctUrls.ToArray(), distinctUrls.Count());
Console.WriteLine("\npress any key to exit the process...");
Console.ReadKey();
}
static async Task Main(string[] args)
{
UrlParserClass parserClass = new UrlParserClass();
FileHelperClass helperClass = new FileHelperClass();
FilterHelperClass filterHelper = new FilterHelperClass();
filterHelper.addFilters("yahoo.com");
filterHelper.addFilters("google.com");
filterHelper.addFilters("bing.com");
filterHelper.addFilters("bingj.com");
filterHelper.addFilters("live.com");
filterHelper.addFilters("w3.org");
filterHelper.addFilters("microsofttranslator.com");
filterHelper.addFilters("wikipedia.org");
filterHelper.addFilters("twitter.com");
filterHelper.addFilters("youtube.com");
filterHelper.addFilters("facebook.com");
filterHelper.addFilters("instagram.com");
filterHelper.addFilters("microsoft.com");
filterHelper.addFilters("giga.de");
filterHelper.addFilters("msn.com");
filterHelper.addFilters("outlook.com");
filterHelper.addFilters("creativecommons.org");
filterHelper.addFilters("trustscam.nl");
filterHelper.addFilters("aol.de");
filterHelper.addFilters("yandex.com");
filterHelper.addFilters("verbraucherschutz.de");
filterHelper.addFilters("whois.com");
filterHelper.addFilters("bingparachute.com");
filterHelper.addFilters("duckduckgo.com");
filterHelper.addFilters(".js");
filterHelper.addFilters(".png");
filterHelper.addFilters(".jpg");
filterHelper.addFilters(".ttf");
filterHelper.addFilters(".pdf");
filterHelper.addFilters(".css");
filterHelper.addFilters(".svg");
filterHelper.addFilters(".sh");
Console.WriteAscii("TAX0R, 2020", Color.LightPink);
Thread.Sleep(TimeSpan.FromSeconds(2));
Console.Clear();
Console.WriteLine("[Important!]: Please drag&drop you'r URL-list into the Application.", Color.Pink);
string toReplace = '"'.ToString();
string input = Console.ReadLine().Replace(toReplace, string.Empty);
Console.WriteLine(input);
Console.Clear();
string[] urls = helperClass.readUrlsFromFile(input);
List <string> scrapedUrls = new List <string>();
foreach (string url in urls)
{
if (filterHelper.isFiltered(url))
{
try
{
Console.WriteLine("[OLD URL]: " + url, Color.Green);
string content = await parserClass.GetUrlContent(url);
string[] foundUrls = parserClass.SearchForUrls(content);
foreach (string foundUrl in foundUrls)
{
if (filterHelper.isFiltered(foundUrl))
{
Console.WriteLine("[NEW URL]: " + foundUrl, Color.LightGreen);
scrapedUrls.Add(foundUrl);
}
}
}
catch (Exception e)
{
Console.WriteLine("[BAD URL]: " + url, Color.Red);
}
}
else
{
Console.WriteLine("[BAD URL]: " + url, Color.Red);
}
}
List <string> distinctUrls = scrapedUrls.Distinct().ToList();
Console.Clear();
Console.WriteLine("[Success!]: URL's we're scraped successfully!");
Console.WriteLine("[Information]: New URL's found: " + scrapedUrls.Count(), Color.LightBlue);
Console.WriteLine("[Information]: Distinct URL's found: " + distinctUrls.Count(), Color.LightPink);
helperClass.saveToFile(distinctUrls.ToArray(), distinctUrls.Count());
Console.WriteLine("\npress any key to exit the process...");
Console.ReadKey();
}