public override void Process(IDroneDataComponent component) { CrunchbaseDataComponent crunchComponent = component as CrunchbaseDataComponent; if (!Object.Equals(crunchComponent, null)) { SaveCompany(crunchComponent.CompanyLocal); } }
public override void Process(IDroneDataComponent component) { CrunchbaseDataComponent comp = component as CrunchbaseDataComponent; if (!Object.Equals(comp.CompanyLocal, null)) { Utility.WriteToLogFile(String.Format("Crunchbase_TestDataRun_{0:M_d_yyyy}", DateTime.Today) + ".log", BuildMessage(comp.CompanyLocal)); } }
public void GetFullCompanyByPermalink() { //@-news.info Crunch crunch = new Crunch(); CrunchbaseDataComponent _dataComponent = new CrunchbaseDataComponent(); _dataComponent.CompanyLocal = crunch.GetFullCompany("What-Is-Your-Monster", new Dig()); Assert.IsNotNull(_dataComponent.CompanyLocal); //Assert.AreEqual("Directi Internet Solutions", cr.records.Registrar); }
public void GetFullCompanyByPermaLinkAndInsertQviaAPI() { //@-news.info Crunch crunch = new Crunch(); CrunchbaseDataComponent _dataComponent = new CrunchbaseDataComponent(); _dataComponent.CompanyLocal = crunch.GetFullCompany("i-App-Creation", new Dig()); crunch.DroneDataSource = new CrunchbaseDataSource(); crunch.DroneDataSource.Process(_dataComponent); Assert.IsNotNull(_dataComponent.CompanyLocal); }
public void GetFullCompanyByPermaLinkAndInsertViaDirectCall() { //@-news.info Crunch crunch = new Crunch(); CrunchbaseDataComponent _dataComponent = new CrunchbaseDataComponent(); _dataComponent.CompanyLocal = crunch.GetFullCompany("i-App-Creation", new Dig()); Drone.QueueProcessor.Datasources.CrunchbaseDataSource cds = new Drone.QueueProcessor.Datasources.CrunchbaseDataSource(); cds.Process(_dataComponent); Assert.IsNotNull(_dataComponent.CompanyLocal); }
public void GetCompanyByIndexAndInsertQueueViaAPI() { List <Company> list = compManager.GetAllCompanies(); int index = list.FindIndex(item => item.permalink == "wetpaint"); Crunch crunch = new Crunch(new CrunchbaseDataSource()); Company co = list.ElementAt(index); CompanyRoot cr = crunch.GetFullCompany(co.permalink, new Dig()); CrunchbaseDataComponent cdc = new CrunchbaseDataComponent(); cdc.CompanyLocal = cr; crunch.DroneDataSource.Process(cdc); Assert.IsNotNull(cr); }
internal void Process(string fileName) { Dig digMgr = new Dig(); List <string> tempLines; while ((tempLines = Utility.ReadLinesFromFile(fileName, 100, true)).Count() > 0) { int maxParallel = XMLUtility.GetTextFromAccountNode(Xml, ProcessorName + "/maxparallel").ConvertStringToInt(1); Parallel.ForEach(tempLines, new ParallelOptions { MaxDegreeOfParallelism = maxParallel }, (tempLine) => { try { if (!String.IsNullOrWhiteSpace(tempLine)) { string[] companyEntry = tempLine.Split('|'); CompanyRoot cr = GetFullCompany(companyEntry[1], digMgr); //Add to MSMQ, add will call receive and then ProcessMessage below when to add to DB asynchronously if (!Object.Equals(null, cr) && !string.IsNullOrEmpty(cr.homepage_url)) { WriteToUsageLogFile("Domain:" + cr.homepage_url, string.Format("Executing {0}.{1}", ComponentTypeName, MethodInfo.GetCurrentMethod().Name), true); CrunchbaseDataComponent cdc = new CrunchbaseDataComponent(); cdc.CompanyLocal = cr; DroneDataSource.Process(cdc); } else { if (Object.Equals(null, cr) && companyEntry.Length < 3) { Utility.AddLineToFile(fileName, tempLine + "|retry"); } } cr = null; } } catch (Exception e) { ExceptionExtensions.LogError(e, "Crunchbase.Process", "Permalink: " + tempLine); } }); } }
public void GetFoundedDate() { //get all entries with 1/1/1900, name and domain name List <string[]> entries = GetAll1900s(); Crunch crunch = new Crunch(new CrunchbaseTestDataSource()); List <Company> _allCompanies = new CompanyManager().GetAllCompanies(); foreach (string[] item in entries) { try { Company co = _allCompanies.Where(com => com.name.Trim().ToLowerInvariant() == item[1].Trim().ToLowerInvariant()).FirstOrDefault(); if (!Object.Equals(co, null)) { CompanyRoot cor = crunch.GetFullCompany(co.permalink, new Dig()); if (!Object.Equals(cor, null)) { cor.homepage_url = cor.homepage_url.Replace("http://", "").Replace("https://", "").Replace("www.", "").TrimEnd(new char[] { '/' }); if (cor.homepage_url == item[0]) { CrunchbaseDataSource ds = new CrunchbaseDataSource(); CrunchbaseDataComponent _dataComponent = crunch.DroneDataComponent as CrunchbaseDataComponent; if (cor.founded_year != null || cor.founded_month != null || cor.founded_day != null) { _dataComponent.CompanyLocal = cor; ds.Process(_dataComponent); } } } } else { //name lookup failed. } } catch (Exception) { } } }
public override void ProcessMessage(object sender, MessageEventArgs args) { bool handled = false; try { string msg = Encoding.UTF8.GetString(args.Message.BodyStream.ToByteArray()); CompanyRoot cr = Utility.DeserializeXMLString <CompanyRoot>(msg); if (!Object.Equals(cr, null)) { WriteToUsageLogFile("Domain:" + cr.homepage_url, string.Format("Executing {0}.{1}", ProcessorName, MethodInfo.GetCurrentMethod().Name), true); CrunchbaseDataComponent dc = new CrunchbaseDataComponent(); dc.CompanyLocal = cr; handled = true; FireMessageProcessingEvent(); QueueComponentDataSource.Process(dc); FireMessageProcessedEvent(); } cr = null; } catch (Exception e) { ExceptionExtensions.LogError(e, "QueueCrunchbase.ProcessMessage"); if (Utility.IsCriticalDBError(e)) { FireShuttingDownEvent(); } if (handled) //if false, processor was never incremented, no need to decrement { FireMessageProcessedEvent(); } } }