public override void Process(IDroneDataComponent component)
{
CrunchbaseDataComponent crunchComponent = component as CrunchbaseDataComponent;
if (!Object.Equals(crunchComponent, null))
{
SaveCompany(crunchComponent.CompanyLocal);
}
}
public override void Process(IDroneDataComponent component)
{
CrunchbaseDataComponent comp = component as CrunchbaseDataComponent;
if (!Object.Equals(comp.CompanyLocal, null))
{
Utility.WriteToLogFile(String.Format("Crunchbase_TestDataRun_{0:M_d_yyyy}", DateTime.Today) + ".log", BuildMessage(comp.CompanyLocal));
}
}
public void GetFullCompanyByPermalink()
{
//@-news.info
Crunch crunch = new Crunch();
CrunchbaseDataComponent _dataComponent = new CrunchbaseDataComponent();
_dataComponent.CompanyLocal = crunch.GetFullCompany("What-Is-Your-Monster", new Dig());
Assert.IsNotNull(_dataComponent.CompanyLocal);
//Assert.AreEqual("Directi Internet Solutions", cr.records.Registrar);
}
public void GetFullCompanyByPermaLinkAndInsertQviaAPI()
{
//@-news.info
Crunch crunch = new Crunch();
CrunchbaseDataComponent _dataComponent = new CrunchbaseDataComponent();
_dataComponent.CompanyLocal = crunch.GetFullCompany("i-App-Creation", new Dig());
crunch.DroneDataSource = new CrunchbaseDataSource();
crunch.DroneDataSource.Process(_dataComponent);
Assert.IsNotNull(_dataComponent.CompanyLocal);
}
public void GetFullCompanyByPermaLinkAndInsertViaDirectCall()
{
//@-news.info
Crunch crunch = new Crunch();
CrunchbaseDataComponent _dataComponent = new CrunchbaseDataComponent();
_dataComponent.CompanyLocal = crunch.GetFullCompany("i-App-Creation", new Dig());
Drone.QueueProcessor.Datasources.CrunchbaseDataSource cds = new Drone.QueueProcessor.Datasources.CrunchbaseDataSource();
cds.Process(_dataComponent);
Assert.IsNotNull(_dataComponent.CompanyLocal);
}
public void GetCompanyByIndexAndInsertQueueViaAPI()
{
List <Company> list = compManager.GetAllCompanies();
int index = list.FindIndex(item => item.permalink == "wetpaint");
Crunch crunch = new Crunch(new CrunchbaseDataSource());
Company co = list.ElementAt(index);
CompanyRoot cr = crunch.GetFullCompany(co.permalink, new Dig());
CrunchbaseDataComponent cdc = new CrunchbaseDataComponent();
cdc.CompanyLocal = cr;
crunch.DroneDataSource.Process(cdc);
Assert.IsNotNull(cr);
}
internal void Process(string fileName)
{
Dig digMgr = new Dig();
List <string> tempLines;
while ((tempLines = Utility.ReadLinesFromFile(fileName, 100, true)).Count() > 0)
{
int maxParallel = XMLUtility.GetTextFromAccountNode(Xml, ProcessorName + "/maxparallel").ConvertStringToInt(1);
Parallel.ForEach(tempLines, new ParallelOptions {
MaxDegreeOfParallelism = maxParallel
}, (tempLine) =>
{
try
{
if (!String.IsNullOrWhiteSpace(tempLine))
{
string[] companyEntry = tempLine.Split('|');
CompanyRoot cr = GetFullCompany(companyEntry[1], digMgr);
//Add to MSMQ, add will call receive and then ProcessMessage below when to add to DB asynchronously
if (!Object.Equals(null, cr) && !string.IsNullOrEmpty(cr.homepage_url))
{
WriteToUsageLogFile("Domain:" + cr.homepage_url, string.Format("Executing {0}.{1}", ComponentTypeName, MethodInfo.GetCurrentMethod().Name), true);
CrunchbaseDataComponent cdc = new CrunchbaseDataComponent();
cdc.CompanyLocal = cr;
DroneDataSource.Process(cdc);
}
else
{
if (Object.Equals(null, cr) && companyEntry.Length < 3)
{
Utility.AddLineToFile(fileName, tempLine + "|retry");
}
}
cr = null;
}
}
catch (Exception e)
{
ExceptionExtensions.LogError(e, "Crunchbase.Process", "Permalink: " + tempLine);
}
});
}
}
public void GetFoundedDate()
{
//get all entries with 1/1/1900, name and domain name
List <string[]> entries = GetAll1900s();
Crunch crunch = new Crunch(new CrunchbaseTestDataSource());
List <Company> _allCompanies = new CompanyManager().GetAllCompanies();
foreach (string[] item in entries)
{
try
{
Company co = _allCompanies.Where(com => com.name.Trim().ToLowerInvariant() == item[1].Trim().ToLowerInvariant()).FirstOrDefault();
if (!Object.Equals(co, null))
{
CompanyRoot cor = crunch.GetFullCompany(co.permalink, new Dig());
if (!Object.Equals(cor, null))
{
cor.homepage_url = cor.homepage_url.Replace("http://", "").Replace("https://", "").Replace("www.", "").TrimEnd(new char[] { '/' });
if (cor.homepage_url == item[0])
{
CrunchbaseDataSource ds = new CrunchbaseDataSource();
CrunchbaseDataComponent _dataComponent = crunch.DroneDataComponent as CrunchbaseDataComponent;
if (cor.founded_year != null || cor.founded_month != null || cor.founded_day != null)
{
_dataComponent.CompanyLocal = cor;
ds.Process(_dataComponent);
}
}
}
}
else
{
//name lookup failed.
}
}
catch (Exception)
{ }
}
}
public override void ProcessMessage(object sender, MessageEventArgs args)
{
bool handled = false;
try
{
string msg = Encoding.UTF8.GetString(args.Message.BodyStream.ToByteArray());
CompanyRoot cr = Utility.DeserializeXMLString <CompanyRoot>(msg);
if (!Object.Equals(cr, null))
{
WriteToUsageLogFile("Domain:" + cr.homepage_url, string.Format("Executing {0}.{1}", ProcessorName, MethodInfo.GetCurrentMethod().Name), true);
CrunchbaseDataComponent dc = new CrunchbaseDataComponent();
dc.CompanyLocal = cr;
handled = true;
FireMessageProcessingEvent();
QueueComponentDataSource.Process(dc);
FireMessageProcessedEvent();
}
cr = null;
}
catch (Exception e)
{
ExceptionExtensions.LogError(e, "QueueCrunchbase.ProcessMessage");
if (Utility.IsCriticalDBError(e))
{
FireShuttingDownEvent();
}
if (handled) //if false, processor was never incremented, no need to decrement
{
FireMessageProcessedEvent();
}
}
}
