static void Main(string[] args) { IWidow info = new ConsoleInformative(); try { string searchQuery = "test"; string searchEngine = "https://google.com"; NameValueCollection search = new NameValueCollection(); Regex linkParser = new Regex(@"\b(?:https?://|www\.)\S+\b", RegexOptions.Compiled | RegexOptions.IgnoreCase); info.Say("Attempting to connect to the site.."); BlackWidow blackWidow = new BlackWidow(searchEngine); info.Success(string.Format("Connected to site, writing HTML to file, and searching {0} with query {1}.", searchEngine, searchQuery)); search.Add("q", searchQuery); } /* * Catch all exceptions and write them * to a file for futher analysis if any * occur during the process. */ catch (Exception e) { var filePath = AppDomain.CurrentDomain.BaseDirectory; info.FatalErr(string.Format("Exception thrown: {0}", e.ToString())); File.WriteAllText(string.Format(@"{0}\errorlog.LOG", filePath), e.ToString()); info.Debug(string.Format("Wrote Exception to file located in {0}", filePath)); } Console.ReadLine(); }
/* * Make a request to the web host in * this case it is Google. */ private static string GetWebInfo(string url) { string logPath = string.Format(@"{0}\html.txt", Path.GetDirectoryName(System.AppDomain.CurrentDomain.BaseDirectory)); string errPath = string.Format(@"{0}\error.txt", Path.GetDirectoryName(System.AppDomain.CurrentDomain.BaseDirectory)); HttpWebRequest requests = (HttpWebRequest)HttpWebRequest.Create(url); requests.ProtocolVersion = HttpVersion.Version10; requests.UserAgent = "A .NET Web Crawler"; IWebProxy proxy = requests.Proxy; IWidow info = new ConsoleInformative(); /* * Used cached credentials to access * proxy if there is one. */ info.Say("Checking if you're behind a proxy"); if (proxy != null) { try { info.Say("Proxy found attempting to login with cached credentials.."); string proxyUri = proxy.GetProxy(requests.RequestUri).ToString(); requests.UseDefaultCredentials = true; requests.Proxy = new WebProxy(proxyUri, false); requests.Proxy.Credentials = System.Net.CredentialCache.DefaultCredentials; } /* * Catch exception if hte cached * credentials fail to load. */ catch (Exception e) { info.FatalErr("Unable to verify cached credentials.."); File.WriteAllText(errPath, e.ToString()); info.Debug("Wrote error to file for further analysis, exiting process.."); } } info.Success("Logged in with cached credentials, continuing process."); WebResponse providedResponse = requests.GetResponse(); Stream stream = providedResponse.GetResponseStream(); StreamReader readInformation = new StreamReader(stream); string htmlOutput = readInformation.ReadToEnd(); File.WriteAllText(logPath, htmlOutput); return(htmlOutput); }