private void BtAllLinksGoogle_Click(object sender, EventArgs e)
{
domain.map.SearchingAllLinks = HttpMap.SearchStatus.Searching;
var wsSearch = new GoogleWebSearcher
{
SearchAll = true,
Site = domain.Domain
};
wsSearch.SearcherLinkFoundEvent += wsSearch_SearcherLinkFoundEvent;
wsSearch.SearcherEndEvent += delegate
{
domain.map.SearchingAllLinks = HttpMap.SearchStatus.Finished;
Invoke(new MethodInvoker(delegate
{
Program.LogThis(new Log(Log.ModuleType.Crawling,
@"Finishing the links extraction of " + domain.Domain, Log.LogType.debug));
Program.FormMainInstance.UpdateBottomPanel(domain);
}));
};
wsSearch.GetCustomLinks("site:" + domain.Domain);
Program.LogThis(new Log(Log.ModuleType.Crawling, "Extracting links of " + domain.Domain, Log.LogType.debug));
AddLog("Google crawling");
}
/// <summary>
/// Perform the DNS search through Google Web
/// </summary>
private void GoogleWebSearch()
{
GoogleWebSearcher googleSearcher = null;
try
{
googleSearcher = new GoogleWebSearcher
{
SearchAll = true,
cSafeSearch = GoogleWebSearcher.SafeSearch.off,
FirstSeen = GoogleWebSearcher.FirstSeenGoogle.AnyTime,
LocatedInRegion = GoogleWebSearcher.Region.AnyRegion,
WriteInLanguage = GoogleWebSearcher.Language.AnyLanguage
};
var currentResults = new List <string>();
googleSearcher.SearcherLinkFoundEvent +=
delegate(object sender, EventsThreads.ThreadListDataFoundEventArgs e)
{
var searcher = (GoogleWebSearcher)sender;
foreach (var item in e.Data)
{
if (CheckToSkip())
{
searcher.Abort();
}
try
{
if (item == null)
{
continue;
}
var url = new Uri((string)item);
var strHost = url.Host;
if (
currentResults.All(
d =>
!string.Equals(d, strHost, StringComparison.CurrentCultureIgnoreCase)))
{
currentResults.Add(strHost);
AddAndLogSubdomainDiscover(strHost);
}
var domain = Program.data.GetDomain(url.Host);
domain.map.AddUrl(url.ToString());
}
catch
{
}
}
};
googleSearcher.SearcherLogEvent += WebSearcherLogEvent;
var endReason = EventsThreads.ThreadEndEventArgs.EndReasonEnum.ErrorFound;
googleSearcher.SearcherEndEvent +=
delegate(object o, EventsThreads.ThreadEndEventArgs e) { endReason = e.EndReason; };
var strSearchString = $"site:{strDomain}";
int resultsNumber;
do
{
if (CheckToSkip())
{
return;
}
resultsNumber = currentResults.Count;
googleSearcher.GetCustomLinks(strSearchString);
googleSearcher.Join();
CheckEndReason(endReason, currentResults, strSearchString, 32);
} while (endReason == EventsThreads.ThreadEndEventArgs.EndReasonEnum.LimitReached &&
resultsNumber != currentResults.Count);
}
catch (ThreadAbortException)
{
}
finally
{
googleSearcher?.Abort();
}
}