static void Main(string[] args)
{
#region Helper Instances
ProductsGenerator productsGenerator = new ProductsGenerator();
CompanyGenerator companyGenerator = new CompanyGenerator();
PeopleGenerator peopleGenerator = new PeopleGenerator();
FinancialGenerator financialGenerator = new FinancialGenerator();
ServiceProviderGenerator serviceprovidergenerator = new ServiceProviderGenerator();
JavaScriptSerializer ser = new JavaScriptSerializer();
DownloadFile myDownload = new DownloadFile();
int num = 0;
int AllNum = 0;
int ThreadNum = 20;
#endregion
if (!Directory.Exists("crunchbase"))
Directory.CreateDirectory("crunchbase");
#region Companies
String companies = myDownload.DownloadParentFileAndGetParentInfo("http://api.crunchbase.com/v/1/companies.js", "company", "crunchbase" + Path.DirectorySeparatorChar + "companies.js");
Console.Write("Parsing company names...");
List<cbCompanyObject> ParsedCompanies = companyGenerator.GetCompanyNames(companies);
Console.WriteLine("done");
Console.Write("Getting each company info...");
if (!Directory.Exists("crunchbase" + Path.DirectorySeparatorChar + "company"))
Directory.CreateDirectory("crunchbase" + Path.DirectorySeparatorChar + "company");
num = 0;
AllNum = ParsedCompanies.Count;
foreach (cbCompanyObject company in ParsedCompanies)
{
#region Get the JSON Results for every company and output it to a file...
//create download task
DownloadChildFileTask atask = new DownloadChildFileTask() { sFilePathDiff = "company", slink = company.permalink, sWebPath = "http://api.crunchbase.com/v/1/company/" };
myDownload.cdChildInfoTasks.Add(atask);
if (num % ThreadNum == 0 || num == AllNum)
{
myDownload.RunChildFileDownloadTask();
}
//myDownload.DownloadChildFile("http://api.crunchbase.com/v/1/company/", company.permalink, "company");
num++;
myDownload.ShowDownloadProgress(num, AllNum, "DownLoad companies:");
#endregion
}
Console.WriteLine("done");
#endregion
#region people
String people = myDownload.DownloadParentFileAndGetParentInfo("http://api.crunchbase.com/v/1/people.js", "peoples", "crunchbase" + Path.DirectorySeparatorChar + "people.js");
Console.Write("Parsing peoples names...");
List<cbPeoplesObject> parsedPeoples = peopleGenerator.GetPeoples(people);
Console.WriteLine("done");
Console.Write("Getting each persons info...");
//string jsonStream;
if (!Directory.Exists("crunchbase" + Path.DirectorySeparatorChar + "person"))
Directory.CreateDirectory("crunchbase" + Path.DirectorySeparatorChar + "person");
num = 0;
AllNum = parsedPeoples.Count;
foreach (cbPeoplesObject person in parsedPeoples)
{
DownloadChildFileTask atask = new DownloadChildFileTask() { sFilePathDiff = "person", slink = person.permalink, sWebPath = "http://api.crunchbase.com/v/1/person/" };
myDownload.cdChildInfoTasks.Add(atask);
if (num % ThreadNum == 0 || num == AllNum)
{
myDownload.RunChildFileDownloadTask();
}
num++;
myDownload.ShowDownloadProgress(num, AllNum, "DownLoad persons:");
}
Console.WriteLine("done");
#endregion
#region products
String products = myDownload.DownloadParentFileAndGetParentInfo("http://api.crunchbase.com/v/1/products.js", "products", "crunchbase" + Path.DirectorySeparatorChar + "products.js");
Console.Write("Parsing products names...");
List<cbProductsObject> parsedProducts = productsGenerator.GetProducts(products);
Console.WriteLine("done");
Console.Write("Getting each products info...");
if (!Directory.Exists("crunchbase" + Path.DirectorySeparatorChar + "product"))
Directory.CreateDirectory("crunchbase" + Path.DirectorySeparatorChar + "product");
num = 0;
AllNum = parsedProducts.Count;
foreach (cbProductsObject product in parsedProducts)
{
DownloadChildFileTask atask = new DownloadChildFileTask() { sFilePathDiff = "product", slink = product.permalink, sWebPath = "http://api.crunchbase.com/v/1/product/" };
myDownload.cdChildInfoTasks.Add(atask);
if (num % ThreadNum == 0 || num == AllNum)
{
myDownload.RunChildFileDownloadTask();
}
//myDownload.DownloadChildFile("http://api.crunchbase.com/v/1/product/", product.permalink, "product");
num++;
myDownload.ShowDownloadProgress(num, AllNum, "DownLoad products:");
}
Console.WriteLine("done");
#endregion
#region financial-organizations
String financialorganizations = myDownload.DownloadParentFileAndGetParentInfo("http://api.crunchbase.com/v/1/financial-organizations.js", "financial-organizations", "crunchbase" + Path.DirectorySeparatorChar + "financial-organizations.js");
Console.Write("Parsing financial-organizations names...");
List<cbFinancialObject> parsedFinancial = financialGenerator.GetFinancials(financialorganizations);
Console.WriteLine("done");
Console.Write("Getting each financial-organization info...");
if (!Directory.Exists("crunchbase" + Path.DirectorySeparatorChar + "financial-organization"))
Directory.CreateDirectory("crunchbase" + Path.DirectorySeparatorChar + "financial-organization");
num = 0;
AllNum = parsedFinancial.Count;
foreach (cbFinancialObject financial in parsedFinancial)
{
DownloadChildFileTask atask = new DownloadChildFileTask() { sFilePathDiff = "financial-organization", slink = financial.permalink, sWebPath = "http://api.crunchbase.com/v/1/financial-organization/" };
myDownload.cdChildInfoTasks.Add(atask);
if (num % ThreadNum == 0 || num == AllNum)
{
myDownload.RunChildFileDownloadTask();
}
// myDownload.DownloadChildFile("http://api.crunchbase.com/v/1/financial-organization/", financial.permalink, "financial-organization");
num++;
myDownload.ShowDownloadProgress(num, AllNum, "DownLoad financial-organizations:");
}
Console.WriteLine("done");
#endregion
#region service-providers
String serviceproviders = myDownload.DownloadParentFileAndGetParentInfo("http://api.crunchbase.com/v/1/service-providers.js", "service-provider", "crunchbase" + Path.DirectorySeparatorChar + "service-providers.js");
Console.Write("Parsing service-provider names...");
List<cbServiceproviderObject> parsedServiceProviders = serviceprovidergenerator.GetFinancials(serviceproviders);
Console.WriteLine("done");
Console.Write("Getting each service-provider info...");
if (!Directory.Exists("crunchbase" + Path.DirectorySeparatorChar + "service-provider"))
Directory.CreateDirectory("crunchbase" + Path.DirectorySeparatorChar + "service-provider");
num = 0;
AllNum = parsedServiceProviders.Count;
foreach (cbServiceproviderObject servicep in parsedServiceProviders)
{
DownloadChildFileTask atask = new DownloadChildFileTask() { sFilePathDiff = "service-provider", slink = servicep.permalink, sWebPath = "http://api.crunchbase.com/v/1/service-provider/" };
myDownload.cdChildInfoTasks.Add(atask);
if (num % ThreadNum == 0 || num == AllNum)
{
myDownload.RunChildFileDownloadTask();
}
// myDownload.DownloadChildFile("http://api.crunchbase.com/v/1/service-provider/", servicep.permalink, "service-provider");
num++;
myDownload.ShowDownloadProgress(num, AllNum, "DownLoad service-providers:");
}
Console.WriteLine("done");
while (true)
{
Console.WriteLine("All download are done,Please Input exit");
string cmd = Console.ReadLine();
if (cmd == "exit")
{
break;
}
}
#endregion
}