public void PostManyCrunchbase() { //post data for (int i = 0; i < 30; i++) { string result = string.Empty; requestPost = (HttpWebRequest)WebRequest.Create("http://localhost/BIdataApi/api/crunchbase"); requestPost.Method = "POST"; requestPost.ContentType = "application/json"; requestPost.UseDefaultCredentials = true; CompanyRoot cr = new CompanyRoot(); cr.permalink = "test-company" + i.ToString(); cr.name = "test company" + i.ToString(); cr.homepage_url = "testcompany.com"; cr.records = new Records { CertificateType = "None", SSLIssuer = "None", DNSHost = "GoDaddy", Registrar = "GoDaddy", EmailHost = "None", WebHost = "GoDaddy" }; string requestData = new JavaScriptSerializer().Serialize(cr); byte[] data = Encoding.UTF8.GetBytes(requestData); using (Stream dataStream = requestPost.GetRequestStream()) dataStream.Write(data, 0, data.Length); HttpWebResponse response = requestPost.GetResponse() as HttpWebResponse; result = new StreamReader(response.GetResponseStream()).ReadToEnd(); } }
private static string GetInvestors(CompanyRoot companyRoot) { List <string> investors = new List <string>(); foreach (var round in companyRoot.funding_rounds) { foreach (var inv in round.investments) { if (!Object.Equals(inv.company, null) && !string.IsNullOrEmpty(inv.company.name) && !investors.Exists(d => d == inv.company.name)) { investors.Add(inv.company.name); } if (!Object.Equals(inv.financial_org, null) && !string.IsNullOrEmpty(inv.financial_org.name) && !investors.Exists(d => d == inv.financial_org.name)) { investors.Add(inv.financial_org.name); } if (!Object.Equals(inv.person, null) && !string.IsNullOrEmpty(inv.person.first_name) && !investors.Exists(d => d == inv.person.first_name)) { investors.Add(string.Format("{0} {1}", inv.person.first_name, inv.person.last_name)); } } } if (Object.Equals(null, investors) || investors.Count <= 0) { return("None"); } else { return(string.Join(",", investors.OrderBy(item => item).ToList())); } }
internal CompanyRoot GetFullCompany(string permalinkName, Dig digMgr) { CompanyRoot company = compManager.GetFullCompany(permalinkName); if (!Object.Equals(company, null) && !Object.Equals(null, company.homepage_url)) { Records r = new Records(); company.homepage_url = Utility.CleanUrl(company.homepage_url); IPAddress ip = digMgr.GetIPAddress(company.homepage_url); if (!Object.Equals(null, ip)) { company.ip_address = ip.ToString(); } SSLCert cert = digMgr.GetSSLVerification(company.homepage_url); if (!Object.Equals(cert, null)) { r.SSLIssuer = cert.FixedName; r.CertificateType = cert.SubjectType; } r.WebHost = digMgr.GetWebHostName(company.homepage_url); r.EmailHost = digMgr.GetEmailHostName(company.homepage_url); r.DNSHost = digMgr.GetDNSHostName(company.homepage_url); r.Registrar = digMgr.GetRegistrar(company.homepage_url); company.records = r; } return(company); }
//allow nulls, the proc should NOT overwrite current data if a null is passed in private void SaveCompany(CompanyRoot companyRoot) { try { string investors = GetInvestors(companyRoot); DateTime dt = DateTime.MinValue; companyRoot.founded_year = companyRoot.founded_year ?? 1900; companyRoot.founded_year = companyRoot.founded_year.Value < 1753 ? 1900 : companyRoot.founded_year; companyRoot.founded_month = companyRoot.founded_month ?? 1; companyRoot.founded_month = companyRoot.founded_month.Value > 12 ? 12 : companyRoot.founded_month; companyRoot.founded_day = companyRoot.founded_day ?? 1; DateTime.TryParse(string.Format("{0}-{1}-{2}", companyRoot.founded_year.Value, companyRoot.founded_month.Value, companyRoot.founded_day.Value), out dt); DataFactory.ExecuteNonQuery("SmallBusinessTrackingInsUpd", new KeyValuePair <string, object>("@Domain", companyRoot.homepage_url), new KeyValuePair <string, object>("@CompanyName", companyRoot.name.Trim()), new KeyValuePair <string, object>("@DateFounded", dt.Date), new KeyValuePair <string, object>("@WebHost", companyRoot.records.WebHost.Trim()), new KeyValuePair <string, object>("@EmailHost", companyRoot.records.EmailHost.Trim()), new KeyValuePair <string, object>("@DNSHost", companyRoot.records.DNSHost.Trim()), new KeyValuePair <string, object>("@Registrar", companyRoot.records.Registrar.Trim()), new KeyValuePair <string, object>("@SSLIssuer", companyRoot.records.SSLIssuer.Trim()), new KeyValuePair <string, object>("@CertificateType", companyRoot.records.CertificateType.Trim()), new KeyValuePair <string, object>("@NumberOfEmployees", companyRoot.number_of_employees ?? 0), new KeyValuePair <string, object>("@TotalFunding", Object.Equals(null, companyRoot.total_money_raised) ? "None" : companyRoot.total_money_raised.Trim()), new KeyValuePair <string, object>("@Investors", investors)); } catch (Exception e) { if (e.Message.Contains("deadlocked")) { SaveCompany(companyRoot); ExceptionExtensions.LogInformation("CrunchbaseDataSource.SaveCompany()", "Deadlock encountered, trying again"); } else if (e.Message.Contains("Timeout expired")) { Thread.Sleep(5000); SaveCompany(companyRoot); } else { ExceptionExtensions.LogError(e, "CrunchbaseDataSource.SaveCompany", "Name: " + companyRoot.name); //if tempdb full or other critical db error, re-throw if (Utility.IsCriticalDBError(e)) { throw; } } } }
public CompanyRoot Create(CompanyRoot value) { try { _queueManager.AddToQueue(Utility.SerializeToXMLString <CompanyRoot>(value), "Crunchbase " + value.permalink); } catch (Exception e) { ExceptionExtensions.LogError(e, "CrunchbaseService.Create", value.permalink); } return(value); }
// GET api/crunchbase/companyName public CompanyRoot Get(string id) { CompanyRoot cr = _crunchService.Get(id); if (Object.Equals(null, cr)) { var resp = new HttpResponseMessage(HttpStatusCode.NotFound) { Content = new StringContent(string.Format("No company with company name = {0}", id)), ReasonPhrase = "Company Not Found" }; throw new HttpResponseException(resp); } return(cr); }
private void SendCompany(CompanyRoot companyRoot) { //CreatePostRequest Uri apiuri = new Uri(XMLUtility.GetTextFromAccountNode(Xml, "apiuri")); //send try { HttpStatusCode code = SendRequest(apiuri, companyRoot, true); } catch (Exception e) { ExceptionExtensions.LogError(e, "CrunchbaseDataSource.SendRequest", companyRoot.permalink); } }
public void GetCompanyByIndexAndInsertQueueViaAPI() { List <Company> list = compManager.GetAllCompanies(); int index = list.FindIndex(item => item.permalink == "wetpaint"); Crunch crunch = new Crunch(new CrunchbaseDataSource()); Company co = list.ElementAt(index); CompanyRoot cr = crunch.GetFullCompany(co.permalink, new Dig()); CrunchbaseDataComponent cdc = new CrunchbaseDataComponent(); cdc.CompanyLocal = cr; crunch.DroneDataSource.Process(cdc); Assert.IsNotNull(cr); }
// POST api/crunchbase public HttpResponseMessage Post([FromBody] CompanyRoot value) { HttpResponseMessage response; if (!Object.Equals(null, value)) { CompanyRoot cr = _crunchService.Create(value); response = Request.CreateResponse(HttpStatusCode.Created); response.Headers.Location = new Uri(Url.Link("DefaultApi", new { permalink = cr.permalink })); } else { response = Request.CreateResponse(HttpStatusCode.BadRequest); } return(response); }
internal void Process(string fileName) { Dig digMgr = new Dig(); List <string> tempLines; while ((tempLines = Utility.ReadLinesFromFile(fileName, 100, true)).Count() > 0) { int maxParallel = XMLUtility.GetTextFromAccountNode(Xml, ProcessorName + "/maxparallel").ConvertStringToInt(1); Parallel.ForEach(tempLines, new ParallelOptions { MaxDegreeOfParallelism = maxParallel }, (tempLine) => { try { if (!String.IsNullOrWhiteSpace(tempLine)) { string[] companyEntry = tempLine.Split('|'); CompanyRoot cr = GetFullCompany(companyEntry[1], digMgr); //Add to MSMQ, add will call receive and then ProcessMessage below when to add to DB asynchronously if (!Object.Equals(null, cr) && !string.IsNullOrEmpty(cr.homepage_url)) { WriteToUsageLogFile("Domain:" + cr.homepage_url, string.Format("Executing {0}.{1}", ComponentTypeName, MethodInfo.GetCurrentMethod().Name), true); CrunchbaseDataComponent cdc = new CrunchbaseDataComponent(); cdc.CompanyLocal = cr; DroneDataSource.Process(cdc); } else { if (Object.Equals(null, cr) && companyEntry.Length < 3) { Utility.AddLineToFile(fileName, tempLine + "|retry"); } } cr = null; } } catch (Exception e) { ExceptionExtensions.LogError(e, "Crunchbase.Process", "Permalink: " + tempLine); } }); } }
public void GetFoundedDate() { //get all entries with 1/1/1900, name and domain name List <string[]> entries = GetAll1900s(); Crunch crunch = new Crunch(new CrunchbaseTestDataSource()); List <Company> _allCompanies = new CompanyManager().GetAllCompanies(); foreach (string[] item in entries) { try { Company co = _allCompanies.Where(com => com.name.Trim().ToLowerInvariant() == item[1].Trim().ToLowerInvariant()).FirstOrDefault(); if (!Object.Equals(co, null)) { CompanyRoot cor = crunch.GetFullCompany(co.permalink, new Dig()); if (!Object.Equals(cor, null)) { cor.homepage_url = cor.homepage_url.Replace("http://", "").Replace("https://", "").Replace("www.", "").TrimEnd(new char[] { '/' }); if (cor.homepage_url == item[0]) { CrunchbaseDataSource ds = new CrunchbaseDataSource(); CrunchbaseDataComponent _dataComponent = crunch.DroneDataComponent as CrunchbaseDataComponent; if (cor.founded_year != null || cor.founded_month != null || cor.founded_day != null) { _dataComponent.CompanyLocal = cor; ds.Process(_dataComponent); } } } } else { //name lookup failed. } } catch (Exception) { } } }
public CompanyRoot Get(string domain) { CompanyRoot cr = null; DataSet ds; try { ds = DataFactory.GetDataSetByName("CrunchbaseGetByDomain", new KeyValuePair <string, object>("@DomainName", domain)); if (ds != null && ds.Tables.Count > 0 && ds.Tables[0].Rows.Count > 0) { cr = new CompanyRoot(); DataRow dr = ds.Tables[0].Rows[0]; cr.homepage_url = dr["Domain"].ToString(); cr.name = dr["CompanyName"].ToString(); cr.permalink = cr.name.Replace(" ", "-"); cr.crunchbase_url = "http://api.crunchbase.com/v/1/company/" + cr.permalink + ".js"; DateTime dateFounded = Convert.ToDateTime(dr["DateFounded"]); cr.founded_day = dateFounded.Day; cr.founded_month = dateFounded.Month; cr.founded_year = dateFounded.Year; cr.records = new Records { WebHost = dr["WebHost"].ToString() , DNSHost = dr["DNSHost"].ToString() , EmailHost = dr["EmailHost"].ToString() , SSLIssuer = dr["SSLIssuer"].ToString() , CertificateType = dr["CertificateType"].ToString() , Registrar = dr["Registrar"].ToString() }; cr.number_of_employees = dr["NumberOfEmployees"].ToString().ConvertStringToInt(0); cr.total_money_raised = dr["TotalFunding"].ToString(); } } catch (Exception) { //todo: log error } return(cr); }
/// <summary> /// Get full company from crunchbase w/retry if it fails /// </summary> /// <param name="permalinkName"></param> /// <param name="retry"></param> /// <returns></returns> public CompanyRoot GetFullCompany(string permalinkName, int retry = 0) { string requestText = string.Format("{1}/company/{0}.js?api_key={2}", permalinkName, ApiUrl, ApiKey); CompanyRoot cr = new CompanyRoot(); try { cr = Request.Deserialize <CompanyRoot>(Request.ExecuteAnonymousWebRequest(requestText), VerboseLogging); if (Object.Equals(cr, null)) { RetryFullCompany(permalinkName, retry); } } catch (Exception) { RetryFullCompany(permalinkName, retry); } return(cr); }
public override void ProcessMessage(object sender, MessageEventArgs args) { bool handled = false; try { string msg = Encoding.UTF8.GetString(args.Message.BodyStream.ToByteArray()); CompanyRoot cr = Utility.DeserializeXMLString <CompanyRoot>(msg); if (!Object.Equals(cr, null)) { WriteToUsageLogFile("Domain:" + cr.homepage_url, string.Format("Executing {0}.{1}", ProcessorName, MethodInfo.GetCurrentMethod().Name), true); CrunchbaseDataComponent dc = new CrunchbaseDataComponent(); dc.CompanyLocal = cr; handled = true; FireMessageProcessingEvent(); QueueComponentDataSource.Process(dc); FireMessageProcessedEvent(); } cr = null; } catch (Exception e) { ExceptionExtensions.LogError(e, "QueueCrunchbase.ProcessMessage"); if (Utility.IsCriticalDBError(e)) { FireShuttingDownEvent(); } if (handled) //if false, processor was never incremented, no need to decrement { FireMessageProcessedEvent(); } } }