public void PostManyCrunchbase()
{
//post data
for (int i = 0; i < 30; i++)
{
string result = string.Empty;
requestPost = (HttpWebRequest)WebRequest.Create("http://localhost/BIdataApi/api/crunchbase");
requestPost.Method = "POST";
requestPost.ContentType = "application/json";
requestPost.UseDefaultCredentials = true;
CompanyRoot cr = new CompanyRoot();
cr.permalink = "test-company" + i.ToString();
cr.name = "test company" + i.ToString();
cr.homepage_url = "testcompany.com";
cr.records = new Records {
CertificateType = "None", SSLIssuer = "None", DNSHost = "GoDaddy", Registrar = "GoDaddy", EmailHost = "None", WebHost = "GoDaddy"
};
string requestData = new JavaScriptSerializer().Serialize(cr);
byte[] data = Encoding.UTF8.GetBytes(requestData);
using (Stream dataStream = requestPost.GetRequestStream())
dataStream.Write(data, 0, data.Length);
HttpWebResponse response = requestPost.GetResponse() as HttpWebResponse;
result = new StreamReader(response.GetResponseStream()).ReadToEnd();
}
}
private static string GetInvestors(CompanyRoot companyRoot)
{
List <string> investors = new List <string>();
foreach (var round in companyRoot.funding_rounds)
{
foreach (var inv in round.investments)
{
if (!Object.Equals(inv.company, null) && !string.IsNullOrEmpty(inv.company.name) && !investors.Exists(d => d == inv.company.name))
{
investors.Add(inv.company.name);
}
if (!Object.Equals(inv.financial_org, null) && !string.IsNullOrEmpty(inv.financial_org.name) && !investors.Exists(d => d == inv.financial_org.name))
{
investors.Add(inv.financial_org.name);
}
if (!Object.Equals(inv.person, null) && !string.IsNullOrEmpty(inv.person.first_name) && !investors.Exists(d => d == inv.person.first_name))
{
investors.Add(string.Format("{0} {1}", inv.person.first_name, inv.person.last_name));
}
}
}
if (Object.Equals(null, investors) || investors.Count <= 0)
{
return("None");
}
else
{
return(string.Join(",", investors.OrderBy(item => item).ToList()));
}
}
internal CompanyRoot GetFullCompany(string permalinkName, Dig digMgr)
{
CompanyRoot company = compManager.GetFullCompany(permalinkName);
if (!Object.Equals(company, null) && !Object.Equals(null, company.homepage_url))
{
Records r = new Records();
company.homepage_url = Utility.CleanUrl(company.homepage_url);
IPAddress ip = digMgr.GetIPAddress(company.homepage_url);
if (!Object.Equals(null, ip))
{
company.ip_address = ip.ToString();
}
SSLCert cert = digMgr.GetSSLVerification(company.homepage_url);
if (!Object.Equals(cert, null))
{
r.SSLIssuer = cert.FixedName;
r.CertificateType = cert.SubjectType;
}
r.WebHost = digMgr.GetWebHostName(company.homepage_url);
r.EmailHost = digMgr.GetEmailHostName(company.homepage_url);
r.DNSHost = digMgr.GetDNSHostName(company.homepage_url);
r.Registrar = digMgr.GetRegistrar(company.homepage_url);
company.records = r;
}
return(company);
}
//allow nulls, the proc should NOT overwrite current data if a null is passed in
private void SaveCompany(CompanyRoot companyRoot)
{
try
{
string investors = GetInvestors(companyRoot);
DateTime dt = DateTime.MinValue;
companyRoot.founded_year = companyRoot.founded_year ?? 1900;
companyRoot.founded_year = companyRoot.founded_year.Value < 1753 ? 1900 : companyRoot.founded_year;
companyRoot.founded_month = companyRoot.founded_month ?? 1;
companyRoot.founded_month = companyRoot.founded_month.Value > 12 ? 12 : companyRoot.founded_month;
companyRoot.founded_day = companyRoot.founded_day ?? 1;
DateTime.TryParse(string.Format("{0}-{1}-{2}", companyRoot.founded_year.Value, companyRoot.founded_month.Value, companyRoot.founded_day.Value), out dt);
DataFactory.ExecuteNonQuery("SmallBusinessTrackingInsUpd",
new KeyValuePair <string, object>("@Domain", companyRoot.homepage_url),
new KeyValuePair <string, object>("@CompanyName", companyRoot.name.Trim()),
new KeyValuePair <string, object>("@DateFounded", dt.Date),
new KeyValuePair <string, object>("@WebHost", companyRoot.records.WebHost.Trim()),
new KeyValuePair <string, object>("@EmailHost", companyRoot.records.EmailHost.Trim()),
new KeyValuePair <string, object>("@DNSHost", companyRoot.records.DNSHost.Trim()),
new KeyValuePair <string, object>("@Registrar", companyRoot.records.Registrar.Trim()),
new KeyValuePair <string, object>("@SSLIssuer", companyRoot.records.SSLIssuer.Trim()),
new KeyValuePair <string, object>("@CertificateType", companyRoot.records.CertificateType.Trim()),
new KeyValuePair <string, object>("@NumberOfEmployees", companyRoot.number_of_employees ?? 0),
new KeyValuePair <string, object>("@TotalFunding", Object.Equals(null, companyRoot.total_money_raised) ? "None" : companyRoot.total_money_raised.Trim()),
new KeyValuePair <string, object>("@Investors", investors));
}
catch (Exception e)
{
if (e.Message.Contains("deadlocked"))
{
SaveCompany(companyRoot);
ExceptionExtensions.LogInformation("CrunchbaseDataSource.SaveCompany()", "Deadlock encountered, trying again");
}
else if (e.Message.Contains("Timeout expired"))
{
Thread.Sleep(5000);
SaveCompany(companyRoot);
}
else
{
ExceptionExtensions.LogError(e, "CrunchbaseDataSource.SaveCompany", "Name: " + companyRoot.name);
//if tempdb full or other critical db error, re-throw
if (Utility.IsCriticalDBError(e))
{
throw;
}
}
}
}
public CompanyRoot Create(CompanyRoot value)
{
try
{
_queueManager.AddToQueue(Utility.SerializeToXMLString <CompanyRoot>(value), "Crunchbase " + value.permalink);
}
catch (Exception e)
{
ExceptionExtensions.LogError(e, "CrunchbaseService.Create", value.permalink);
}
return(value);
}
// GET api/crunchbase/companyName
public CompanyRoot Get(string id)
{
CompanyRoot cr = _crunchService.Get(id);
if (Object.Equals(null, cr))
{
var resp = new HttpResponseMessage(HttpStatusCode.NotFound)
{
Content = new StringContent(string.Format("No company with company name = {0}", id)),
ReasonPhrase = "Company Not Found"
};
throw new HttpResponseException(resp);
}
return(cr);
}
private void SendCompany(CompanyRoot companyRoot)
{
//CreatePostRequest
Uri apiuri = new Uri(XMLUtility.GetTextFromAccountNode(Xml, "apiuri"));
//send
try
{
HttpStatusCode code = SendRequest(apiuri, companyRoot, true);
}
catch (Exception e)
{
ExceptionExtensions.LogError(e, "CrunchbaseDataSource.SendRequest", companyRoot.permalink);
}
}
public void GetCompanyByIndexAndInsertQueueViaAPI()
{
List <Company> list = compManager.GetAllCompanies();
int index = list.FindIndex(item => item.permalink == "wetpaint");
Crunch crunch = new Crunch(new CrunchbaseDataSource());
Company co = list.ElementAt(index);
CompanyRoot cr = crunch.GetFullCompany(co.permalink, new Dig());
CrunchbaseDataComponent cdc = new CrunchbaseDataComponent();
cdc.CompanyLocal = cr;
crunch.DroneDataSource.Process(cdc);
Assert.IsNotNull(cr);
}
// POST api/crunchbase
public HttpResponseMessage Post([FromBody] CompanyRoot value)
{
HttpResponseMessage response;
if (!Object.Equals(null, value))
{
CompanyRoot cr = _crunchService.Create(value);
response = Request.CreateResponse(HttpStatusCode.Created);
response.Headers.Location = new Uri(Url.Link("DefaultApi", new { permalink = cr.permalink }));
}
else
{
response = Request.CreateResponse(HttpStatusCode.BadRequest);
}
return(response);
}
internal void Process(string fileName)
{
Dig digMgr = new Dig();
List <string> tempLines;
while ((tempLines = Utility.ReadLinesFromFile(fileName, 100, true)).Count() > 0)
{
int maxParallel = XMLUtility.GetTextFromAccountNode(Xml, ProcessorName + "/maxparallel").ConvertStringToInt(1);
Parallel.ForEach(tempLines, new ParallelOptions {
MaxDegreeOfParallelism = maxParallel
}, (tempLine) =>
{
try
{
if (!String.IsNullOrWhiteSpace(tempLine))
{
string[] companyEntry = tempLine.Split('|');
CompanyRoot cr = GetFullCompany(companyEntry[1], digMgr);
//Add to MSMQ, add will call receive and then ProcessMessage below when to add to DB asynchronously
if (!Object.Equals(null, cr) && !string.IsNullOrEmpty(cr.homepage_url))
{
WriteToUsageLogFile("Domain:" + cr.homepage_url, string.Format("Executing {0}.{1}", ComponentTypeName, MethodInfo.GetCurrentMethod().Name), true);
CrunchbaseDataComponent cdc = new CrunchbaseDataComponent();
cdc.CompanyLocal = cr;
DroneDataSource.Process(cdc);
}
else
{
if (Object.Equals(null, cr) && companyEntry.Length < 3)
{
Utility.AddLineToFile(fileName, tempLine + "|retry");
}
}
cr = null;
}
}
catch (Exception e)
{
ExceptionExtensions.LogError(e, "Crunchbase.Process", "Permalink: " + tempLine);
}
});
}
}
public void GetFoundedDate()
{
//get all entries with 1/1/1900, name and domain name
List <string[]> entries = GetAll1900s();
Crunch crunch = new Crunch(new CrunchbaseTestDataSource());
List <Company> _allCompanies = new CompanyManager().GetAllCompanies();
foreach (string[] item in entries)
{
try
{
Company co = _allCompanies.Where(com => com.name.Trim().ToLowerInvariant() == item[1].Trim().ToLowerInvariant()).FirstOrDefault();
if (!Object.Equals(co, null))
{
CompanyRoot cor = crunch.GetFullCompany(co.permalink, new Dig());
if (!Object.Equals(cor, null))
{
cor.homepage_url = cor.homepage_url.Replace("http://", "").Replace("https://", "").Replace("www.", "").TrimEnd(new char[] { '/' });
if (cor.homepage_url == item[0])
{
CrunchbaseDataSource ds = new CrunchbaseDataSource();
CrunchbaseDataComponent _dataComponent = crunch.DroneDataComponent as CrunchbaseDataComponent;
if (cor.founded_year != null || cor.founded_month != null || cor.founded_day != null)
{
_dataComponent.CompanyLocal = cor;
ds.Process(_dataComponent);
}
}
}
}
else
{
//name lookup failed.
}
}
catch (Exception)
{ }
}
}
public CompanyRoot Get(string domain)
{
CompanyRoot cr = null;
DataSet ds;
try
{
ds = DataFactory.GetDataSetByName("CrunchbaseGetByDomain", new KeyValuePair <string, object>("@DomainName", domain));
if (ds != null && ds.Tables.Count > 0 && ds.Tables[0].Rows.Count > 0)
{
cr = new CompanyRoot();
DataRow dr = ds.Tables[0].Rows[0];
cr.homepage_url = dr["Domain"].ToString();
cr.name = dr["CompanyName"].ToString();
cr.permalink = cr.name.Replace(" ", "-");
cr.crunchbase_url = "http://api.crunchbase.com/v/1/company/" + cr.permalink + ".js";
DateTime dateFounded = Convert.ToDateTime(dr["DateFounded"]);
cr.founded_day = dateFounded.Day;
cr.founded_month = dateFounded.Month;
cr.founded_year = dateFounded.Year;
cr.records = new Records {
WebHost = dr["WebHost"].ToString()
, DNSHost = dr["DNSHost"].ToString()
, EmailHost = dr["EmailHost"].ToString()
, SSLIssuer = dr["SSLIssuer"].ToString()
, CertificateType = dr["CertificateType"].ToString()
, Registrar = dr["Registrar"].ToString()
};
cr.number_of_employees = dr["NumberOfEmployees"].ToString().ConvertStringToInt(0);
cr.total_money_raised = dr["TotalFunding"].ToString();
}
}
catch (Exception)
{
//todo: log error
}
return(cr);
}
/// <summary>
/// Get full company from crunchbase w/retry if it fails
/// </summary>
/// <param name="permalinkName"></param>
/// <param name="retry"></param>
/// <returns></returns>
public CompanyRoot GetFullCompany(string permalinkName, int retry = 0)
{
string requestText = string.Format("{1}/company/{0}.js?api_key={2}", permalinkName, ApiUrl, ApiKey);
CompanyRoot cr = new CompanyRoot();
try
{
cr = Request.Deserialize <CompanyRoot>(Request.ExecuteAnonymousWebRequest(requestText), VerboseLogging);
if (Object.Equals(cr, null))
{
RetryFullCompany(permalinkName, retry);
}
}
catch (Exception)
{
RetryFullCompany(permalinkName, retry);
}
return(cr);
}
public override void ProcessMessage(object sender, MessageEventArgs args)
{
bool handled = false;
try
{
string msg = Encoding.UTF8.GetString(args.Message.BodyStream.ToByteArray());
CompanyRoot cr = Utility.DeserializeXMLString <CompanyRoot>(msg);
if (!Object.Equals(cr, null))
{
WriteToUsageLogFile("Domain:" + cr.homepage_url, string.Format("Executing {0}.{1}", ProcessorName, MethodInfo.GetCurrentMethod().Name), true);
CrunchbaseDataComponent dc = new CrunchbaseDataComponent();
dc.CompanyLocal = cr;
handled = true;
FireMessageProcessingEvent();
QueueComponentDataSource.Process(dc);
FireMessageProcessedEvent();
}
cr = null;
}
catch (Exception e)
{
ExceptionExtensions.LogError(e, "QueueCrunchbase.ProcessMessage");
if (Utility.IsCriticalDBError(e))
{
FireShuttingDownEvent();
}
if (handled) //if false, processor was never incremented, no need to decrement
{
FireMessageProcessedEvent();
}
}
}
