private static void Main(string[] args)
{
Uri url = null;
bool goodUrl = false;
while (!goodUrl)
{
Console.ForegroundColor = ConsoleColor.Red;
Console.Write("Enter URL: ");
Console.ForegroundColor = ConsoleColor.Green;
string surl = Console.ReadLine();
goodUrl = Uri.TryCreate(surl, UriKind.Absolute, out url);
if (!goodUrl)
{
Console.ForegroundColor = ConsoleColor.DarkRed;
Console.WriteLine("The URL is not valid, please try again.");
}
}
Console.ForegroundColor = ConsoleColor.Red;
Console.Write("Select Keywords (Seperate With Comma): ");
Console.ForegroundColor = ConsoleColor.Green;
string keywords = Console.ReadLine();
Keywords = keywords.Split(',');
Robot bot = new Robot(url, Robot.MicrosoftInternetExplorer70UserAgent);
bot.MaxProcessorsAllowed = 1;
bot.ProcessSubPages = false;
bot.UriProcessingFinished += new EventHandler<UriProcessingFinishedEventArgs>(bot_UriProcessingFinished);
bot.Scan();
Console.ForegroundColor = ConsoleColor.Green;
Console.WriteLine("Processing Done");
Console.Read();
}
private static void Main(string[] args)
{
Console.ForegroundColor = ConsoleColor.Red;
Console.Write("Enter Domain: ");
Console.ForegroundColor = ConsoleColor.Green;
LookingForDomain = Console.ReadLine();
Console.ForegroundColor = ConsoleColor.Red;
Console.Write("Select Keywords: ");
Console.ForegroundColor = ConsoleColor.Green;
string keywords = Console.ReadLine();
Console.ForegroundColor = ConsoleColor.Red;
Console.Write("Number of Results to Search: ");
Console.ForegroundColor = ConsoleColor.Green;
string sresultsToSearch = Console.ReadLine();
int ressultsToSearch;
if (!Int32.TryParse(sresultsToSearch, out ressultsToSearch))
ressultsToSearch = 200;
for (int i = 0; i < ressultsToSearch; i = i + 10)
{
CurrentlyProcessingStart = i;
Console.ForegroundColor = ConsoleColor.Yellow;
Console.WriteLine("Trying {0} to {1}", CurrentlyProcessingStart, CurrentlyProcessingStart + 10);
Console.ResetColor();
UriBuilder url = new UriBuilder("http", "www.google.com", 80, "search");
url.Query = "num=10&lr=lang_en&safe=off&start=" + i + "&q=" + keywords;
Robot bot = new Robot(url.Uri, Robot.MicrosoftInternetExplorer70UserAgent);
bot.MaxProcessorsAllowed = 1;
bot.ProcessSubPages = false;
bot.UriProcessingFinished += new EventHandler<UriProcessingFinishedEventArgs>(bot_UriProcessingFinished);
bot.Scan();
}
Console.ForegroundColor = ConsoleColor.Red;
Console.Write("Do you want to save these results? ");
Console.ForegroundColor = ConsoleColor.Green;
Console.Beep();
string saveResults = Console.ReadLine();
if (!String.IsNullOrEmpty(saveResults) && saveResults.ToLower()[0] == 'y')
{
SaveFileDialog saveDialog = new SaveFileDialog();
saveDialog.Filter = "Text File (*.txt)|*.txt";
saveDialog.Title = "Where do you want to save the results?";
saveDialog.AddExtension = true;
saveDialog.SupportMultiDottedExtensions = true;
saveDialog.OverwritePrompt = true;
saveDialog.AutoUpgradeEnabled = true;
saveDialog.DefaultExt = "txt";
if (saveDialog.ShowDialog() == DialogResult.OK)
{
using (var writer = new StreamWriter(saveDialog.OpenFile()))
{
writer.WriteLine("Domain: " + LookingForDomain);
writer.WriteLine("Keywords: " + keywords);
writer.WriteLine();
foreach (string result in ResultsFound)
writer.WriteLine(result);
}
}
}
Console.ForegroundColor = ConsoleColor.Green;
Console.WriteLine("Processing Done");
Console.Read();
}
private static void Main(string[] args)
{
SessionKey = Guid.NewGuid();
Errors = new List<string>();
UriFoundCount = 0;
// set number of connections allowed to recommend limit
// <see href="http://support.microsoft.com/kb/821268" />
ServicePointManager.DefaultConnectionLimit = 12 * Environment.ProcessorCount;
Console.ForegroundColor = ConsoleColor.Red;
Console.Write("Enter Domain: ");
Console.ForegroundColor = ConsoleColor.Green;
string url = Console.ReadLine();
if (!url.StartsWith("http://"))
url = "http://" + url;
Console.ForegroundColor = ConsoleColor.Red;
Console.Write("Enter Number of Processors (" + 12 * Environment.ProcessorCount + " recommended): ");
Console.ForegroundColor = ConsoleColor.Green;
string cprocessors = Console.ReadLine();
int processors;
if (!Int32.TryParse(cprocessors, out processors))
processors = 1;
Console.ForegroundColor = ConsoleColor.Cyan;
Console.WriteLine("Starting scan of {0} with {1} thread{2}", url, processors, processors == 1 ? String.Empty : "s");
Console.ResetColor();
using (var dc = new CrawlerDataContext())
{
Session session = new Session {
SessionKey = SessionKey,
ScanDate = DateTime.UtcNow,
Url = url
};
dc.Sessions.InsertOnSubmit(session);
dc.SubmitChanges();
}
Robot bot = new Robot(new Uri(url));
bot.MaxProcessorsAllowed = processors;
bot.UriProcessingFinished += new EventHandler<UriProcessingFinishedEventArgs>(bot_UriProcessingFinished);
bot.UriFound += new EventHandler<UriFoundEventArgs>(bot_UriFound);
bot.Scan();
Console.ForegroundColor = ConsoleColor.Green;
Console.WriteLine("Processing Done");
if (Errors.Count > 0)
{
Console.ForegroundColor = ConsoleColor.Red;
Console.WriteLine("Errors");
Console.ResetColor();
for (int i = 0; i < Errors.Count; i++)
{
if (i % 2 == 0)
Console.ForegroundColor = ConsoleColor.Red;
Console.WriteLine(Errors[i]);
Console.ResetColor();
}
}
Console.Read();
}