예제 #1
0
        public void LoadUserData(IList <Uri> addresses, bool IsTruncateTempDB = true)
        {
            WebHttpCrawlerClient client = new WebHttpCrawlerClient();

            client.SetRequestDetail = (x) =>
            {
                x._HttpWebRequest.AutomaticDecompression = DecompressionMethods.GZip;
                x._HttpWebRequest.Referer = "https://api.stackexchange.com/docs/users";
            };

            WebThreadDesigation t = new WebThreadDesigation(client);

            t.EventWaitHandleName = "WebThreadDesigation";

            StackoverflowPageAnalysis           ana = new StackoverflowPageAnalysis();
            AnalysisThreadDesigation <UserInfo> a   = new AnalysisThreadDesigation <UserInfo>(ana);

            a.EventWaitHandleName = "StackoverflowPageAnalysis";
            DBThreadDesigation d = new DBThreadDesigation();

            d.EventWaitHandleName = "DBThreadDesigation";
            CrawlerSyncObject  syscobject       = new CrawlerSyncObject();
            AnalysisSyncObject anasissyncObject = new AnalysisSyncObject();

            SendImformation infos = new SendImformation();

            infos.ConnectString = "data source=CIA-SH-svr-sis;initial catalog=stackoverflowNew;integrated security=True;";
            //infos.EventhandlerName = "dbo.SuggestionStagging";
            //infos.DestinationtableName = "dbo.SuggestionStagging";

            infos.EventhandlerName     = "UserTemp1_Tinct";
            infos.DestinationtableName = "UserTemp1_Tinct";
            infos.IsWantToMerged       = false;
            infos.MergeSQlSPName       = "";

            EventWaitHandle eventWaitHandle1 = new System.Threading.EventWaitHandle(false, System.Threading.EventResetMode.ManualReset, t.EventWaitHandleName);

            eventWaitHandle1.Reset();
            EventWaitHandle eventWaitHandle2 = new System.Threading.EventWaitHandle(false, System.Threading.EventResetMode.ManualReset, a.EventWaitHandleName);

            eventWaitHandle2.Reset();
            EventWaitHandle eventWaitHandle3 = new System.Threading.EventWaitHandle(false, System.Threading.EventResetMode.ManualReset, d.EventWaitHandleName);

            eventWaitHandle3.Reset();
            EventWaitHandle eventWaitHandle4 = new System.Threading.EventWaitHandle(false, System.Threading.EventResetMode.ManualReset, infos.EventhandlerName);

            eventWaitHandle4.Reset();
            Task.Run(() => { t.CrawlerUris(addresses, 10, syscobject); });
            Task.Run(() => { a.AnalysisResults(t.resultlists, 5, syscobject, anasissyncObject); });
            Task.Run(() =>
            {
                d.SendDispatchDatas <UserInfo>(ref a.datas, 500, ref anasissyncObject, infos,
                                               null);
            });
            eventWaitHandle1.WaitOne();
            eventWaitHandle2.WaitOne();
            eventWaitHandle3.WaitOne();
            eventWaitHandle4.WaitOne();
        }
예제 #2
0
        public void LoadAnswersData(IList<Uri> addresses, bool IsTruncateTempDB = true)
        {

            WebHttpCrawlerClient client = new WebHttpCrawlerClient();

            client.SetRequestDetail = (x) =>
            {
                x._HttpWebRequest.AutomaticDecompression = DecompressionMethods.GZip;
                x._HttpWebRequest.Referer = "https://api.stackexchange.com/docs/answers";
            };

            WebThreadDesigation t = new WebThreadDesigation(client);
            t.EventWaitHandleName = "WebThreadDesigation";

            StackoverflowPageAnalysis ana = new StackoverflowPageAnalysis();
            AnalysisThreadDesigation<AnswerInfo> a = new AnalysisThreadDesigation<AnswerInfo>(ana);
            a.EventWaitHandleName = "StackoverflowPageAnalysis";
            DBThreadDesigation d = new DBThreadDesigation();
            d.EventWaitHandleName = "DBThreadDesigation";
            CrawlerSyncObject syscobject = new CrawlerSyncObject();
            AnalysisSyncObject anasissyncObject = new AnalysisSyncObject();

            SendImformation infos = new SendImformation();
            infos.ConnectString = "data source=CIA-SH-svr-sis;initial catalog=stackoverflowNew;integrated security=True;"; 

            infos.EventhandlerName = "AnswerTemp1_Tinct";
            infos.DestinationtableName = "AnswerTemp1_Tinct";
            infos.IsWantToMerged = false;
            infos.MergeSQlSPName = "";

            EventWaitHandle eventWaitHandle1 = new System.Threading.EventWaitHandle(false, System.Threading.EventResetMode.ManualReset, t.EventWaitHandleName);
            eventWaitHandle1.Reset();
            EventWaitHandle eventWaitHandle2 = new System.Threading.EventWaitHandle(false, System.Threading.EventResetMode.ManualReset, a.EventWaitHandleName);
            eventWaitHandle2.Reset();
            EventWaitHandle eventWaitHandle3 = new System.Threading.EventWaitHandle(false, System.Threading.EventResetMode.ManualReset, d.EventWaitHandleName);
            eventWaitHandle3.Reset();
            EventWaitHandle eventWaitHandle4 = new System.Threading.EventWaitHandle(false, System.Threading.EventResetMode.ManualReset, infos.EventhandlerName);
            eventWaitHandle4.Reset();
            Task.Run(() => { t.CrawlerUris(addresses, 10, syscobject); });
            Task.Run(() => { a.AnalysisResults(t.resultlists, 5, syscobject, anasissyncObject); });
            Task.Run(() =>
            {
                d.SendDispatchDatas<AnswerInfo>(ref a.datas, 500, ref anasissyncObject, infos,
                    null);
            });
            eventWaitHandle1.WaitOne();
            eventWaitHandle2.WaitOne();
            eventWaitHandle3.WaitOne();
            eventWaitHandle4.WaitOne();
        }