예제 #1
0
        private void finalUrlCollectionCampaignScraper(string CheckString, string Account)
        {
            Thread.Sleep(1 * 30 * 1000);
            if (Globals.IsStop_CompanyEmployeeScraperThread)
            {
                return;
            }
            Globals.lstCompanyEmployeeScraperThread.Add(Thread.CurrentThread);
            Globals.lstCompanyEmployeeScraperThread = Globals.lstCompanyEmployeeScraperThread.Distinct().ToList();
            Thread.CurrentThread.IsBackground = true;

            DataSet DS = new DataSet();
            DataTable DT = new DataTable();
            clsDBQueryManager DQ = new clsDBQueryManager();

            try
            {
                DS = DQ.SelectUrl(Account);
                DT = DS.Tables["tb_CampaignScraperURL"];
            }
            catch
            { }
            try
            {
                List<string> numburlpp = new List<string>();
                GlobusHttpHelper HttpHelper = _HttpHelper;
                if (SearchCriteria.starter)
                {

                    if (DT.Rows.Count > 0)
                    {
                        foreach (DataRow item in DT.Rows)
                        {
                            string urltemp = string.Empty;
                            try
                            {
                                string Url = Convert.ToString(item["Url"]);

                                if (Url.Contains("pp_profile_name_link"))
                                {
                                    if (SearchCriteria.starter)
                                    {
                                        urltemp = Url;
                                        numburlpp.Add(urltemp);

                                        Log("[ " + DateTime.Now + " ] => [ " + urltemp + " ]");

                                        Log("[ " + DateTime.Now + " ] => [ Fetching Data From URL ]");
                                        urltemp = urltemp + CheckString;
                                        bool check = CrawlingLinkedInPage(urltemp, ref HttpHelper);
                                        if (!check)
                                        {
                                            string stop = string.Empty;
                                        }
                                    }
                                }
                                else if (Url.Contains("/profile/view?"))
                                {
                                    urltemp = Url;
                                    numburlpp.Add(urltemp);

                                    Log("[ " + DateTime.Now + " ] => [ " + urltemp + " ]");

                                    Log("[ " + DateTime.Now + " ] => [ Fetching Data From URL ]");
                                    urltemp = urltemp + CheckString;
                                    bool check = CrawlingLinkedInPage(urltemp, ref HttpHelper);
                                    if (!check)
                                    {
                                        string stop = string.Empty;
                                    }
                                }

                                try
                                {
                                    urltemp = urltemp + "##";
                                    string Db_Url = string.Empty;
                                    int startindex = urltemp.IndexOf("CampaignScraper");
                                    string start = urltemp.Substring(startindex);
                                    int EndIndex = start.IndexOf("##");
                                    string End = start.Substring(0, EndIndex);
                                    string Rep = End;

                                    urltemp = urltemp.Replace(Rep, string.Empty).Replace("CampaignScraper", string.Empty).Replace("##",string.Empty);

                                    DQ.UpdateCampaignScraperUrl(urltemp);
                                }
                                catch
                                { }

                            }
                            catch
                            { }
                        }
                        Thread.Sleep(30 * 1000);
                        DS = DQ.SelectUrl(Account);
                        DT = DS.Tables["tb_CampaignScraperURL"];
                        if (DT.Rows.Count == 0)
                        {
                            //break;
                        }
                    }
                    else
                    {
                        Thread.Sleep(1 * 30 * 1000);

                        if (queRecordUrl.Count == 0)
                        {
                            Thread.Sleep(1 * 60 * 1000);

                            if (queRecordUrl.Count == 0)
                            {
                                Log("--------------------Find All the Data-------------------");
                                //break;
                            }
                        }
                    }
                }
                Log("[ " + DateTime.Now + " ] => [ PROCESS COMPLETED ]");
                Log("-----------------------------------------------------------------------------------------------------------------------------------");
            }
            catch (Exception ex)
            {

            }
        }