/// <summary>
/// Setup Web Search
/// </summary>
private void SetUpWebSearch()
{
var message =
$"Searching subdomains for {strDomain} in {PanelWebSearcherInformation.EngineToString(searchEngine)}";
Program.LogThis(new Log(Log.ModuleType.WebSearch, message, Log.LogType.debug));
Program.ChangeStatus(message);
}
/// <summary>
/// Perform web search based on the user's choice
/// </summary>
private void SearchWeb()
{
string message = $"Searching subdomains for {strDomain} in {PanelWebSearcherInformation.EngineToString(searchEngine)}";
Program.LogThis(new Log(Log.ModuleType.WebSearch, message, Log.LogType.debug));
Program.ChangeStatus(message);
LinkSearcher searcher = null;
switch (searchEngine)
{
case PanelWebSearcherInformation.Engine.GoogleWeb:
searcher = new GoogleWebSearcher
{
cSafeSearch = GoogleWebSearcher.SafeSearch.off,
FirstSeen = GoogleWebSearcher.FirstSeenGoogle.AnyTime,
LocatedInRegion = GoogleWebSearcher.Region.AnyRegion,
WriteInLanguage = GoogleWebSearcher.Language.AnyLanguage
};
break;
case PanelWebSearcherInformation.Engine.GoogleAPI:
searcher = new GoogleAPISearcher(Program.cfgCurrent.GoogleApiKey, Program.cfgCurrent.GoogleApiCx);
break;
case PanelWebSearcherInformation.Engine.BingWeb:
searcher = new BingWebSearcher
{
LocatedInRegion = BingWebSearcher.Region.AnyRegion,
WriteInLanguage = BingWebSearcher.Language.AnyLanguage
};
break;
case PanelWebSearcherInformation.Engine.BingAPI:
searcher = new BingAPISearcher(Program.cfgCurrent.BingApiKey);
break;
case PanelWebSearcherInformation.Engine.DuckDuckGoWeb:
searcher = new DuckduckgoWebSearcher();
break;
}
try
{
searcher.ItemsFoundEvent += CaptureSearchResults;
searcher.SearcherLogEvent += WebSearcherLogEvent;
CancelIfSkipRequested();
searcher.SearchBySite(this.searchCancelToken, strDomain)
.ContinueWith((e) => SearchEngineFinish(e, searcher.Name, Log.ModuleType.WebSearch))
.Wait();
}
catch (OperationCanceledException)
{
}
}
/// <summary>
/// Adds and logs a subdomain discovery event
/// </summary>
/// <param name="strHost">Discovered subdomain</param>
private void AddAndLogSubdomainDiscover(string strHost)
{
Program.data.AddDomain(strHost,
string.Format("WebSearch, {1} [{0}]", strHost,
PanelWebSearcherInformation.EngineToString(searchEngine)),
MaxRecursion, Program.cfgCurrent);
Program.LogThis(new Log(Log.ModuleType.WebSearch,
$"[{PanelWebSearcherInformation.EngineToString(searchEngine)}] Found subdomain {strHost}",
Log.LogType.debug));
}
/// <summary>
/// max length of the request is 2058 including the GET and HTTP/1.1 words
/// we limit the string to 1900 to leave space for offset, count and filters
/// </summary>
/// <param name="len"></param>
/// <returns></returns>
private bool CheckTotalLength(int len)
{
var res = len > 1900;
if (res)
{
Program.LogThis(new Log(Log.ModuleType.WebSearch,
$"[{PanelWebSearcherInformation.EngineToString(searchEngine)}] Reached limit of 1900 characters in search string.",
Log.LogType.debug));
}
return(res);
}
/// <summary>
/// Checks thread's end reason and logs if the reason was LimitReached
/// </summary>
/// <param name="endReason"></param>
/// <param name="currentResults"></param>
/// <param name="strSearchString"></param>
/// <param name="wordsCountLimit"></param>
private void CheckEndReason(EventsThreads.ThreadEndEventArgs.EndReasonEnum endReason,
IList <string> currentResults, string strSearchString, int wordsCountLimit)
{
if (strSearchString == null)
{
return;
}
if (endReason != EventsThreads.ThreadEndEventArgs.EndReasonEnum.LimitReached)
{
return;
}
var newSearchString = new StringBuilder($"site:{strDomain}");
var wordsCounter = 1;
foreach (var item in currentResults)
{
var strExcludeSite = $" -site:{item}";
if (CheckTotalLength(newSearchString.Length + strExcludeSite.Length))
{
break;
}
newSearchString.Append(strExcludeSite);
if (++wordsCounter != wordsCountLimit)
{
continue;
}
var excludeSubDomains = string.Empty;
var currentIndex = currentResults.IndexOf(item);
for (var i = currentIndex; i < currentResults.Count; i++)
{
excludeSubDomains += currentResults[i] + " ";
}
LogLimit(excludeSubDomains);
break;
}
strSearchString = newSearchString.ToString();
Program.LogThis(new Log(Log.ModuleType.WebSearch,
$"[{PanelWebSearcherInformation.EngineToString(searchEngine)}] Searching again with restricted sites, search string: {strSearchString}",
Log.LogType.debug));
}
/// <summary>
/// Logs that length limit was reached
/// </summary>
/// <param name="excludeSubdomains"></param>
private void LogLimit(string excludeSubdomains)
{
Program.LogThis(new Log(Log.ModuleType.WebSearch,
$"[{PanelWebSearcherInformation.EngineToString(searchEngine)}] Reached limit of 32 words in search string, the next subdomains can't be used to exclude them: {excludeSubdomains}",
Log.LogType.debug));
}
/// <summary>
/// Perform web search based on the user's choice
/// </summary>
private void SearchWeb()
{
string message = $"Searching subdomains for {strDomain} in {PanelWebSearcherInformation.EngineToString(searchEngine)}";
Program.LogThis(new Log(Log.ModuleType.WebSearch, message, Log.LogType.debug));
Program.ChangeStatus(message);
WebSearcher searcher = null;
switch (searchEngine)
{
case PanelWebSearcherInformation.Engine.GoogleWeb:
searcher = new GoogleWebSearcher
{
SearchAll = true,
cSafeSearch = GoogleWebSearcher.SafeSearch.off,
FirstSeen = GoogleWebSearcher.FirstSeenGoogle.AnyTime,
LocatedInRegion = GoogleWebSearcher.Region.AnyRegion,
WriteInLanguage = GoogleWebSearcher.Language.AnyLanguage
};
break;
case PanelWebSearcherInformation.Engine.GoogleAPI:
searcher = new GoogleAPISearcher(Program.cfgCurrent.GoogleApiKey, Program.cfgCurrent.GoogleApiCx)
{
SearchAll = true
};
break;
case PanelWebSearcherInformation.Engine.BingWeb:
searcher = new BingWebSearcher
{
LocatedInRegion = BingWebSearcher.Region.AnyRegion,
SearchAll = true,
WriteInLanguage = BingWebSearcher.Language.AnyLanguage
};
break;
case PanelWebSearcherInformation.Engine.BingAPI:
searcher = new BingAPISearcher(Program.cfgCurrent.BingApiKey);
break;
case PanelWebSearcherInformation.Engine.DuckDuckGoWeb:
searcher = new DuckduckgoWebSearcher();
break;
}
try
{
searcher.SearcherLinkFoundEvent += CaptureSearchResults;
searcher.SearcherLogEvent += WebSearcherLogEvent;
string strSearchString = $"site:{strDomain}";
if (CheckToSkip())
{
return;
}
searcher.GetCustomLinks(strSearchString);
searcher.Join();
}
catch (ThreadAbortException)
{
}
finally
{
searcher?.Abort();
}
}
