/// <summary>
/// Checks if specified domain is allowed by the current permission.
/// </summary>
/// <param name="domain">Domain to check.</param>
/// <returns>True if passed domain is allowed.</returns>
public bool IsDomainAllowed(string domain)
{
if (string.IsNullOrEmpty(domain))
{
throw new ArgumentException("Domain is null or empty!", "domain");
}
return(Domains == null || Domains.Contains(domain, StringComparer.OrdinalIgnoreCase));
}
public bool IsRobot(LogDataItem logDataItem)
{
if (Domains.Contains(logDataItem.UserDomain.Trim()) ||
IsRobotUserAgent(logDataItem.UserAgent.Trim()))
{
return(true);
}
return(false);
}
public async Task ManageInternalCrawl(InBoundLinkCheckerViewModel model, List <Examine.SearchResult> PublishedPages)
{
// Instantiate Crawler Variables
var LinksAvailableToCrawl = true;
var TaskCount = 0;
var TaskID = 0;
var TaskStatus = new Dictionary <int, string>();
var TaskList = new Dictionary <int, Task <CrawlerModel> >();
// Keep going while there are still umbraco pages to crawl
while (LinksAvailableToCrawl)
{
try
{
// While there are less than 8 async tasks running and at least 1 page to crawl.
while (TaskCount < 8 && PublishedPages.Count > 0)
{
TaskCount++;
TaskID++;
TaskList.Add(TaskID, Task.Run(() => ProcessPage(model, PublishedPages.Take(1).ToList())));
PublishedPages.RemoveRange(0, 1);
TaskStatus.Add(TaskID, "Started");
}
// If there are no pages left to crawl after assigning tasks, set LinksAvailableToCrawl to false to end the while loop after this iteration
if (PublishedPages.Count() <= 0 && TaskCount == 0)
{
LinksAvailableToCrawl = false;
}
// Instantiate a List to store the results of the aysnc tasks.
var ResultsModelList = new List <CrawlerModel>();
// Foreach task in the list, if one is completed, gather its result and log its status as completed.
foreach (var Task in TaskList)
{
if (Task.Value.IsCompleted)
{
ResultsModelList.Add(Task.Value.Result);
TaskStatus[Task.Key] = "Completed";
}
}
// Create a tempory list to log which task keys should be removed.
var KeysToRemove = new List <int>();
// Foreach task in the status list, if it is logged as completed, remove the task from the task list and add its key to the keystoremove list.
foreach (var Task in TaskStatus)
{
if (Task.Value == "Completed")
{
TaskList.Remove(Task.Key);
TaskCount--;
KeysToRemove.Add(Task.Key);
}
}
foreach (var Key in KeysToRemove)
{
TaskStatus.Remove(Key);
}
// Instantiate the Collections to store results in.
var ResultsDictionary = cache["ResultsDictionary"] as Dictionary <string, ContentModel>;
if (ResultsDictionary == null)
{
ResultsDictionary = new Dictionary <string, ContentModel>();
}
var BrokenLinks = cache["BrokenLinks"] as List <BrokenPageModel>;
if (BrokenLinks == null)
{
BrokenLinks = new List <BrokenPageModel>();
}
var LinksFound = cache["LinksFound"] as List <string>;
if (LinksFound == null)
{
LinksFound = new List <string>();
}
var Domains = cache["Domains"] as List <string>;
if (Domains == null)
{
Domains = new List <string>();
}
// For each result model in the resultmodellist, if it is not null, process the models results into the results collections.
foreach (var ResultModel in ResultsModelList)
{
if (ResultModel != null)
{
model.CrawledLinks += ResultModel.CrawledLinks;
foreach (var item in ResultModel.BrokenLinks)
{
if (!BrokenLinks.Contains(item))
{
BrokenLinks.Add(item);
}
}
foreach (var item in ResultModel.ResultsDictionary)
{
if (!ResultsDictionary.Keys.Contains(item.Key))
{
ResultsDictionary.Add(item.Key, item.Value);
}
}
foreach (var item in ResultModel.LinksFound)
{
if (!LinksFound.Contains(item))
{
LinksFound.Add(item);
}
}
foreach (var item in ResultModel.Domains)
{
if (!Domains.Contains(item))
{
Domains.Add(item);
}
}
}
}
// make a note of the number of pages that have been crawled and verified.
model.IndexedPagesTotal = ResultsDictionary.Count();
StoreModelInCache(model);
// store the results in the cache
StoreResultsInCache(ResultsDictionary, LinksFound, BrokenLinks, Domains);
}
catch (Exception ex)
{
// If an Exception occurs then stop the crawl and store the results up till now.
model.DataBeingGenerated = false;
model.CachedDataAvailable = true;
model.ErrorOccured = ex.InnerException.Message;
StoreModelInCache(model);
break;
}
}
// Now the Crawl has ended, set the view model booleans to let the view know that data is no longer being generated, and their is cached data to view.
model.DataBeingGenerated = false;
model.CachedDataAvailable = true;
StoreModelInCache(model);
}
/// <summary>
/// Add Ip.
/// </summary>
/// <param name="ip"></param>
/// <param name="source"></param>
/// <param name="domainSource"></param>
public void AddIP(string ip, string source, string domainSource, int MaxRecursion, bool doptr)
{
ip = ip.Trim();
if (isIPv6(ip))
{
ip = ParseIPV6(ip);
}
if (!Ips.Items.Any(I => I.Ip.ToLower() == ip.ToLower()))
{
if (isPublicIP(ip))
{
var isInNetrange = Project.IsIpInNetrange(ip);
if (!isInNetrange)
{
var host = string.Empty;
try
{
host = Dns.GetHostEntry(ip).HostName;
if (Program.data.Project.LstNetRange.Count == 0)
{
if (Program.data.Project.Domain != null)
{
if (!IsMainDomainOrAlternative(host))
{
if (Program.data.Project.AlternativeDomains.Select(S => host.Contains(S.ToString())).Count() == 0)
{
string[] arrDom = host.Split(new char[] { '.' });
if (arrDom.Count() > 1)
{
string auxFinalDom = arrDom[arrDom.Length - 2] + "." + arrDom[arrDom.Length - 1];
Program.data.Project.AlternativeDomains.Add(auxFinalDom);
MessageBox.Show("IP address associated to " + Program.data.Project.Domain + " belongs to a Netrange of " + auxFinalDom + ". It is going to be added as an alternative domain.", "Information", MessageBoxButtons.OK, MessageBoxIcon.Information);
}
}
}
}
}
}
catch (Exception)
{
}
if (IsMainDomainOrAlternative(host))
{
var netrange = Project.GetNetrange(ip);
if (netrange != null)
{
Project.LstNetRange.Add(netrange);
#if PLUGINS
Thread tPluginOnNetrange = new Thread(new ParameterizedThreadStart(Program.data.plugins.OnNewNetrange));
tPluginOnNetrange.IsBackground = true;
object[] oNetRange = new object[] { new object[] { netrange.from, netrange.to } };
tPluginOnNetrange.Start(oNetRange);
#endif
if (!Program.cfgCurrent.ScanNetranges255 || Project.GetIpsOfNetrange(netrange) <= 255)
{
List <string> lstIps = netrange.GenerateIpsOfNetrange();
Program.LogThis(new Log(Log.ModuleType.IPRangeSearch, "Netrange with " + lstIps.Count.ToString() + " IPs", Log.LogType.low));
Thread tAddIps = new Thread(new ParameterizedThreadStart(AddIpListAsync));
tAddIps.IsBackground = true;
tAddIps.Priority = ThreadPriority.Lowest;
tAddIps.Start(lstIps);
}
}
}
}
}
var ipItem = new IPsItem(ip, source);
Ips.Items.Add(ipItem);
// OnNewIP
#if PLUGINS
Thread tPluginOnIP = new Thread(new ParameterizedThreadStart(Program.data.plugins.OnNewIP));
tPluginOnIP.IsBackground = true;
object[] oIP = new object[] { new object[] { ip } };
tPluginOnIP.Start(oIP);
#endif
if (MaxRecursion <= 0)
{
OnChangeEvent(null);
return;
}
List <string> domains;
if (doptr)
{
if (domainSource != null)
{
if (Program.cfgCurrent.UseAllDns)
{
domains = new List <string>();
List <string> dnsServers = DNSUtil.GetNSServer(resolver, domainSource, DNSUtil.GetLocalNSServer().First().ToString());
foreach (string dns in dnsServers)
{
OnLog(null, new EventsThreads.ThreadStringEventArgs(string.Format("Making reverse resolution to IP: {0} Using DNS server: {1}", ip, dns)));
foreach (var domain in DNSUtil.GetHostNames(resolver, ip, dns).Where(domain => !domains.Contains(domain)))
{
domains.Add(domain);
}
}
}
else
{
var dnsserver = DNSUtil.GetNSServer(resolver, domainSource);
OnLog(null, new EventsThreads.ThreadStringEventArgs(string.Format("Making reverse resolution to IP: {0} Using DNS server: {1}", ip, dnsserver)));
domains = DNSUtil.GetHostNames(resolver, ip, dnsserver);
}
}
else
{
domains = DNSUtil.GetHostNames(resolver, ip);
}
foreach (var domain in domains)
{
AddResolution(domain, ip, string.Format("{0} > DNS reverse resolution [{1}]", GetIpSource(ip), domain), MaxRecursion - 1, Program.cfgCurrent, true);
}
}
OnChangeEvent(null);
}
}