private void btAllLinksBing_Click(object sender, EventArgs e)
{
domain.map.SearchingAllLinks = HttpMap.SearchStatus.Searching;
var wsSearch = new BingWebSearcher
{
SearchAll = true,
Site = domain.Domain
};
wsSearch.SearcherLinkFoundEvent += wsSearch_SearcherLinkFoundEvent;
wsSearch.SearcherEndEvent += delegate
{
domain.map.SearchingAllLinks = HttpMap.SearchStatus.Finished;
Invoke(new MethodInvoker(delegate
{
Program.LogThis(new Log(Log.ModuleType.Crawling,
"Finishing the links extraction of " + domain.Domain, Log.LogType.debug));
Program.FormMainInstance.UpdateBottomPanel(domain);
}));
};
wsSearch.GetCustomLinks("site:" + domain.Domain);
Program.LogThis(new Log(Log.ModuleType.Crawling, "Extracting links of " + domain.Domain, Log.LogType.debug));
AddLog("Bing crawling");
}
/// <summary>
/// Event for search link found.
/// </summary>
/// <param name="ip"></param>
/// <param name="bingSearcher"></param>
/// <param name="currentResults"></param>
private void SerchLinkWebBingEvent(string ip, BingWebSearcher bingSearcher, List <string> currentResults)
{
bingSearcher.SearcherLinkFoundEvent +=
delegate(object sender, EventsThreads.ThreadListDataFoundEventArgs e)
{
var op = Partitioner.Create(e.Data);
var po = new ParallelOptions();
if (Program.cfgCurrent.ParallelDnsQueries != 0)
{
po.MaxDegreeOfParallelism = Program.cfgCurrent.ParallelDnsQueries;
}
Parallel.ForEach(op, po, delegate(object item, ParallelLoopState loopState)
{
if (CheckToSkip())
{
loopState.Stop();
}
try
{
var url = new Uri((string)item);
if (currentResults.Any(d => d.ToLower() == url.Host.ToLower()))
{
return;
}
currentResults.Add(url.Host);
var isInBaseDomain = false;
try
{
if (Dns.GetHostAddresses(url.Host).Any(IP => IP.ToString() == ip))
{
isInBaseDomain = true;
}
}
catch
{
}
if (!isInBaseDomain)
{
return;
}
var source = $"{Program.data.GetIpSource(ip)} > Bing IP Search [{url.Host}]";
Program.data.AddResolution(url.Host, ip, source, MaxRecursion,
Program.cfgCurrent, true);
Program.LogThis(new Log(Log.ModuleType.IPBingSearch,
$"[{PanelSearchIPBing.EngineToString(searchIPEngine)}] Found domain {url.Host} in IP {ip}",
Log.LogType.medium));
}
catch
{
}
}
);
};
bingSearcher.SearcherLogEvent += IpBingSearcherLogEvent;
bingSearcher.GetCustomLinks($"ip:{ip}");
bingSearcher.Join();
}
/// <summary>
/// Perform the search through Bing Web
/// </summary>
private void BingWebSearch()
{
BingWebSearcher bingSearcher = null;
try
{
bingSearcher = new BingWebSearcher
{
LocatedInRegion = BingWebSearcher.Region.AnyRegion,
SearchAll = true,
WriteInLanguage = BingWebSearcher.Language.AnyLanguage
};
var currentResults = new List <string>();
bingSearcher.SearcherLinkFoundEvent +=
delegate(object sender, EventsThreads.ThreadListDataFoundEventArgs e)
{
var searcher = (BingWebSearcher)sender;
foreach (var item in e.Data)
{
if (CheckToSkip())
{
searcher.Abort();
}
try
{
var url = new Uri((string)item);
var strHost = url.Host;
if (
currentResults.All(
D => !string.Equals(D, strHost, StringComparison.CurrentCultureIgnoreCase)))
{
currentResults.Add(strHost);
AddAndLogSubdomainDiscover(strHost);
}
var domain = Program.data.GetDomain(url.Host);
domain.map.AddUrl(url.ToString());
}
catch
{
}
}
};
bingSearcher.SearcherLogEvent += WebSearcherLogEvent;
var endReason = EventsThreads.ThreadEndEventArgs.EndReasonEnum.ErrorFound;
bingSearcher.SearcherEndEvent +=
delegate(object o, EventsThreads.ThreadEndEventArgs e) { endReason = e.EndReason; };
var strSearchString = $"site:{strDomain}";
int nroResultados;
do
{
if (CheckToSkip())
{
return;
}
nroResultados = currentResults.Count;
bingSearcher.GetCustomLinks(strSearchString);
bingSearcher.Join();
CheckEndReason(endReason, currentResults, strSearchString, 49);
} while (endReason == EventsThreads.ThreadEndEventArgs.EndReasonEnum.LimitReached &&
nroResultados != currentResults.Count);
}
catch (ThreadAbortException)
{
}
finally
{
bingSearcher?.Abort();
}
}
