Beispiel #1
0
        static void Main(string[] args)
        {
            #region Helper Instances

            ProductsGenerator productsGenerator = new ProductsGenerator();
            CompanyGenerator companyGenerator = new CompanyGenerator();
            PeopleGenerator peopleGenerator = new PeopleGenerator();
            FinancialGenerator financialGenerator = new FinancialGenerator();
            ServiceProviderGenerator serviceprovidergenerator = new ServiceProviderGenerator();

            JavaScriptSerializer ser = new JavaScriptSerializer();

            DownloadFile myDownload = new DownloadFile();
            int num = 0;
            int AllNum = 0;
            int ThreadNum = 20;
            #endregion

            if (!Directory.Exists("crunchbase"))
                Directory.CreateDirectory("crunchbase");

            #region Companies

            String companies = myDownload.DownloadParentFileAndGetParentInfo("http://api.crunchbase.com/v/1/companies.js", "company", "crunchbase" + Path.DirectorySeparatorChar + "companies.js");

            Console.Write("Parsing company names...");
            List<cbCompanyObject> ParsedCompanies = companyGenerator.GetCompanyNames(companies);
            Console.WriteLine("done");

            Console.Write("Getting each company info...");
            if (!Directory.Exists("crunchbase" + Path.DirectorySeparatorChar + "company"))
                Directory.CreateDirectory("crunchbase" + Path.DirectorySeparatorChar + "company");

            num = 0;
            AllNum = ParsedCompanies.Count;
            foreach (cbCompanyObject company in ParsedCompanies)
            {
                #region Get the JSON Results for every company and output it to a file...

                //create download task
                DownloadChildFileTask atask = new DownloadChildFileTask() { sFilePathDiff = "company", slink = company.permalink, sWebPath = "http://api.crunchbase.com/v/1/company/" };
                myDownload.cdChildInfoTasks.Add(atask);

                if (num % ThreadNum == 0 || num == AllNum)
                {
                    myDownload.RunChildFileDownloadTask();
                }

                //myDownload.DownloadChildFile("http://api.crunchbase.com/v/1/company/", company.permalink, "company");

                num++;
                myDownload.ShowDownloadProgress(num, AllNum, "DownLoad companies:");

                #endregion
            }
            Console.WriteLine("done");
            #endregion

            #region people

            String people = myDownload.DownloadParentFileAndGetParentInfo("http://api.crunchbase.com/v/1/people.js", "peoples", "crunchbase" + Path.DirectorySeparatorChar + "people.js");

            Console.Write("Parsing peoples names...");
            List<cbPeoplesObject> parsedPeoples = peopleGenerator.GetPeoples(people);
            Console.WriteLine("done");

            Console.Write("Getting each persons info...");
            //string jsonStream;

            if (!Directory.Exists("crunchbase" + Path.DirectorySeparatorChar + "person"))
                Directory.CreateDirectory("crunchbase" + Path.DirectorySeparatorChar + "person");

            num = 0;
            AllNum = parsedPeoples.Count;
            foreach (cbPeoplesObject person in parsedPeoples)
            {
                DownloadChildFileTask atask = new DownloadChildFileTask() { sFilePathDiff = "person", slink = person.permalink, sWebPath = "http://api.crunchbase.com/v/1/person/" };
                myDownload.cdChildInfoTasks.Add(atask);
                if (num % ThreadNum == 0 || num == AllNum)
                {
                    myDownload.RunChildFileDownloadTask();
                }

                num++;
                myDownload.ShowDownloadProgress(num, AllNum, "DownLoad persons:");

            }
            Console.WriteLine("done");

            #endregion

            #region products

            String products = myDownload.DownloadParentFileAndGetParentInfo("http://api.crunchbase.com/v/1/products.js", "products", "crunchbase" + Path.DirectorySeparatorChar + "products.js");

            Console.Write("Parsing products names...");
            List<cbProductsObject> parsedProducts = productsGenerator.GetProducts(products);
            Console.WriteLine("done");

            Console.Write("Getting each products info...");

            if (!Directory.Exists("crunchbase" + Path.DirectorySeparatorChar + "product"))
                Directory.CreateDirectory("crunchbase" + Path.DirectorySeparatorChar + "product");

            num = 0;
            AllNum = parsedProducts.Count;
            foreach (cbProductsObject product in parsedProducts)
            {

                DownloadChildFileTask atask = new DownloadChildFileTask() { sFilePathDiff = "product", slink = product.permalink, sWebPath = "http://api.crunchbase.com/v/1/product/" };
                myDownload.cdChildInfoTasks.Add(atask);

                if (num % ThreadNum == 0 || num == AllNum)
                {
                    myDownload.RunChildFileDownloadTask();
                }

                //myDownload.DownloadChildFile("http://api.crunchbase.com/v/1/product/", product.permalink, "product");

                num++;
                myDownload.ShowDownloadProgress(num, AllNum, "DownLoad products:");
            }
            Console.WriteLine("done");

            #endregion

            #region financial-organizations

            String financialorganizations = myDownload.DownloadParentFileAndGetParentInfo("http://api.crunchbase.com/v/1/financial-organizations.js", "financial-organizations", "crunchbase" + Path.DirectorySeparatorChar + "financial-organizations.js");

            Console.Write("Parsing financial-organizations names...");
            List<cbFinancialObject> parsedFinancial = financialGenerator.GetFinancials(financialorganizations);
            Console.WriteLine("done");

            Console.Write("Getting each financial-organization info...");

            if (!Directory.Exists("crunchbase" + Path.DirectorySeparatorChar + "financial-organization"))
                Directory.CreateDirectory("crunchbase" + Path.DirectorySeparatorChar + "financial-organization");
            num = 0;
            AllNum = parsedFinancial.Count;
            foreach (cbFinancialObject financial in parsedFinancial)
            {

                DownloadChildFileTask atask = new DownloadChildFileTask() { sFilePathDiff = "financial-organization", slink = financial.permalink, sWebPath = "http://api.crunchbase.com/v/1/financial-organization/" };
                myDownload.cdChildInfoTasks.Add(atask);

                if (num % ThreadNum == 0 || num == AllNum)
                {
                    myDownload.RunChildFileDownloadTask();
                }

                //  myDownload.DownloadChildFile("http://api.crunchbase.com/v/1/financial-organization/", financial.permalink, "financial-organization");

                num++;
                myDownload.ShowDownloadProgress(num, AllNum, "DownLoad financial-organizations:");
            }
            Console.WriteLine("done");

            #endregion

            #region service-providers

            String serviceproviders = myDownload.DownloadParentFileAndGetParentInfo("http://api.crunchbase.com/v/1/service-providers.js", "service-provider", "crunchbase" + Path.DirectorySeparatorChar + "service-providers.js");

            Console.Write("Parsing service-provider names...");
            List<cbServiceproviderObject> parsedServiceProviders = serviceprovidergenerator.GetFinancials(serviceproviders);
            Console.WriteLine("done");

            Console.Write("Getting each service-provider info...");

            if (!Directory.Exists("crunchbase" + Path.DirectorySeparatorChar + "service-provider"))
                Directory.CreateDirectory("crunchbase" + Path.DirectorySeparatorChar + "service-provider");

            num = 0;
            AllNum = parsedServiceProviders.Count;
            foreach (cbServiceproviderObject servicep in parsedServiceProviders)
            {

                DownloadChildFileTask atask = new DownloadChildFileTask() { sFilePathDiff = "service-provider", slink = servicep.permalink, sWebPath = "http://api.crunchbase.com/v/1/service-provider/" };
                myDownload.cdChildInfoTasks.Add(atask);

                if (num % ThreadNum == 0 || num == AllNum)
                {
                    myDownload.RunChildFileDownloadTask();
                }

                // myDownload.DownloadChildFile("http://api.crunchbase.com/v/1/service-provider/", servicep.permalink, "service-provider");

                num++;
                myDownload.ShowDownloadProgress(num, AllNum, "DownLoad service-providers:");
            }
            Console.WriteLine("done");

            while (true)
            {
                Console.WriteLine("All download are done,Please Input exit");
                string cmd = Console.ReadLine();
                if (cmd == "exit")
                {
                    break;
                }
            }
            #endregion
        }