internal CompanyRoot GetFullCompany(string permalinkName, Dig digMgr)
{
CompanyRoot company = compManager.GetFullCompany(permalinkName);
if (!Object.Equals(company, null) && !Object.Equals(null, company.homepage_url))
{
Records r = new Records();
company.homepage_url = Utility.CleanUrl(company.homepage_url);
IPAddress ip = digMgr.GetIPAddress(company.homepage_url);
if (!Object.Equals(null, ip))
{
company.ip_address = ip.ToString();
}
SSLCert cert = digMgr.GetSSLVerification(company.homepage_url);
if (!Object.Equals(cert, null))
{
r.SSLIssuer = cert.FixedName;
r.CertificateType = cert.SubjectType;
}
r.WebHost = digMgr.GetWebHostName(company.homepage_url);
r.EmailHost = digMgr.GetEmailHostName(company.homepage_url);
r.DNSHost = digMgr.GetDNSHostName(company.homepage_url);
r.Registrar = digMgr.GetRegistrar(company.homepage_url);
company.records = r;
}
return(company);
}
public void DigWebHostName_Other()
{
//resolver will cache these, so remove the cache lookup
//dig does a friendly name lookup as well, that can be removed for a VERY slight speed increase over a large #
Dig dig = new Dig();
List <string> domainList = new List <string>();
List <string> webhostList = new List <string>();
List <string> dnshostList = new List <string>();
//domainList.Add("mprconsultinghk.com");
domainList.Add("weebly.com");
domainList.Add("kraftymoms.com");
domainList.Add("paranique.com");
domainList.Add("casaartandphoto.com");
foreach (var item in domainList)
{
webhostList.Add(dig.GetWebHostName(item));
}
foreach (var item in domainList)
{
dnshostList.Add(dig.GetDNSHostName(item));
}
}
public void DigWebHostName()
{
Dig dig = new Dig();
string record = dig.GetWebHostName("godaddy.com");
Assert.IsTrue(record == "Self Hosted");
}
private void GetAllWebHosts()
{
MarketShareDataSource mds = DroneDataSource as MarketShareDataSource;
if (!Object.Equals(null, mds))
{
using (Dig dig = new Dig())
{
int maxParallel = XMLUtility.GetTextFromAccountNode(Xml, ProcessorName + "/maxparallel").ConvertStringToInt(1);
Parallel.ForEach(mds.GetAllCompanies(MarketShareTypeBitMaskEnum.WebHost, XMLUtility.GetIntFromAccountNode(Xml, ProcessorName + "/recordcount"))
, new ParallelOptions {
MaxDegreeOfParallelism = maxParallel
}
, (company, state) =>
{
try
{
if (Context.ShuttingDown)
{
state.Break();
}
MarketShareDataType marketType = new MarketShareDataType();
string url = Utility.CleanUrl(company.Uri.ToString());
if (!String.IsNullOrWhiteSpace(url))
{
WriteToUsageLogFile("Domain:" + url, string.Format("Executing {0}.{1}", ComponentTypeName, MethodInfo.GetCurrentMethod().Name), true);
marketType.Value = dig.GetWebHostName(url);
marketType.DomainId = company.DomainId;
marketType.SampleDate = company.DomainAttributes["SampleDate"];
marketType.TypeId = (int)MarketShareDataTypeEnum.WebHost;
marketType.BitMaskId = (int)MarketShareTypeBitMaskEnum.WebHost;
marketType.UniqueID = g;
if (!string.IsNullOrWhiteSpace(marketType.Value))
{
MarketShareDataComponent dc = new MarketShareDataComponent();
dc.MarketShareType = marketType;
DroneDataSource.Process(dc);
}
else
{
Utility.WriteToLogFile(String.Format("SmallBiz_NoWebHost_{0:M_d_yyyy}", DateTime.Today) + ".log", string.Format("Domain: ,{0}", url));
}
}
}
catch (Exception e)
{
ExceptionExtensions.LogError(e, "WebHost.GetAllWebHostsParallel", "Domain: " + company.Uri.ToString());
}
});
}
}
}
public void FindCompanyInLookups()
{
Dig dig = new Dig();
string foundName = dig.GetRegistrar("1computer.info");
Assert.AreEqual("Network Solutions", foundName);
//self hosted
foundName = dig.GetEmailHostName("fash-art.com");
Assert.AreEqual("Self Hosted", foundName);
foundName = dig.GetEmailHostName("blooclick.com");
Assert.AreEqual("ovh systems", foundName.ToLower());
//email
foundName = dig.GetEmailHostName("sawyerit.com");
Assert.AreEqual("Go Daddy", foundName);
//not found, use record
foundName = dig.GetDNSHostName("travellution.com");
Assert.AreEqual("technorail.com", foundName);
//found, use company name
foundName = dig.GetDNSHostName("godaddy.com");
Assert.AreEqual("Go Daddy", foundName);
//no SSL issuer
foundName = dig.GetCompanyFromRecordName(dig.GetSSLVerification("cybergeekshop.com").IssuerName, "cybergeekshop.com", DigTypeEnum.SSL);
Assert.AreEqual("None", foundName);
//webhost (split AS name with -)
foundName = dig.GetWebHostName("cybergeekshop.com");
Assert.AreEqual("Unified Layer", foundName);
foundName = dig.GetWebHostName("microteksystems.net");
Assert.AreEqual("SoftLayer", foundName);
//webhost (splitting AS Name without -)
foundName = dig.GetWebHostName("eatads.com");
Assert.AreEqual("Amazon", foundName);
}
public void DigURL_PerformanceTest()
{
//resolver will cache these, so remove the cache lookup
//dig does a friendly name lookup as well, that can be removed for a VERY slight speed increase over a large #
Dig dig = new Dig();
List <string> domainList = new List <string>();
List <string> webhostList = new List <string>();
List <string> errorList = new List <string>();
domainList.Add("mprconsultinghk.com");
domainList.Add("tnipresents.com");
domainList.Add("kraftymoms.com");
domainList.Add("paranique.com");
domainList.Add("eatads.com");
domainList.Add("travellution.com");
domainList.Add("bee.com");
domainList.Add("yahoo.com");
domainList.Add("google.com");
domainList.Add("microsoft.com");
DateTime endTime;
DateTime startTime = DateTime.Now;
string header = string.Empty;
int statCode = 0;
startTime = DateTime.Now;
Parallel.For(0, 100, (i) =>
{
foreach (var item in domainList)
{
try
{
CheckHead(ref statCode, ref header, item);
}
catch (Exception e)
{
if (e.Message.Contains("timed out"))
{
try
{
CheckHead(ref statCode, ref header, item);
}
catch (Exception ex)
{
if (ex.Message.Contains("timed out"))
{
errorList.Add("headrequest timed out");
}
}
}
else
{
errorList.Add(e.Message);
}
}
try
{
//add host to list
if (statCode == 301)
{
webhostList.Add(dig.GetWebHostName(CleanUrl(header)));
}
else
{
webhostList.Add(dig.GetWebHostName(item));
}
}
catch (Exception)
{
webhostList.Add("webhost timed out");
}
}
});
endTime = DateTime.Now;
TimeSpan elapsedTime1 = endTime.Subtract(startTime);
List <string> timeoutList = webhostList.Where(item => item == "headrequest timed out").ToList();
List <string> webtimeoutList = webhostList.Where(item => item == "webhost timed out").ToList();
}
public void DigWebHostName()
{
Dig dig = new Dig();
string record = dig.GetWebHostName("netflix.com");
}
public void WebHostLookup_FromZoneSample()
{
Dig dig = new Dig();
//copy the file because we are going to delete as we go instead of holding a million records in memory
string masterFile = "PortSample.txt";
string tempFile = "temp_" + masterFile;
string tempLine = string.Empty;
List <string> webhostList = new List <string>();
List <string> errorList = new List <string>();
string header = string.Empty;
int statCode = 0;
int counter = 0;
// Ensure that the target does not exist.
File.Copy(masterFile, tempFile, true);
try
{
//read lines from tempfile & parse it
using (StreamReader r = new StreamReader(tempFile))
{
// Use while != null pattern for loop
while ((tempLine = r.ReadLine()) != null)
{
if (!String.IsNullOrWhiteSpace(tempLine))
{
//we have a line, parse the tabs and call webhost lookups
//string[] seperator = { "\t" };
string[] seperator = { "|" };
string[] cols = tempLine.Split(seperator, StringSplitOptions.RemoveEmptyEntries);
counter++;
if (cols.Length > 0)
{
IPAddress addr = null;
addr = dig.GetIPAddress(Utility.CleanUrl(cols[0]));
if (addr != null)
{
header = string.Empty;
}
statCode = 0;
try
{
CheckHead(ref statCode, ref header, cols[0]);
}
catch (Exception)
{
errorList.Add("headrequest timed out");
}
try
{
//add host to list
if (statCode == 301)
{
webhostList.Add(dig.GetWebHostName(Utility.CleanUrl(header)));
}
else
{
webhostList.Add(dig.GetWebHostName(cols[0]));
}
}
catch (Exception)
{
webhostList.Add("webhost timed out");
}
}
}
}
List <string> timeoutList = webhostList.Where(item => item == "headrequest timed out").ToList();
List <string> webtimeoutList = webhostList.Where(item => item == "webhost timed out").ToList();
}
File.Delete(tempFile);
}
catch (Exception)
{
throw;
}
//do lookup
//write data
}
public void GetProviders_NoRegisteredProcessors_ReturnsEmptyCollection()
{
Dig dig = Dig.Instance;
dig.GetWebHostName("mybenchcoach.com");
}
