コード例 #1
0
        private async Task SetPageProcessingStateAsync(int pageNumber, ProcessingState state, string errorMessage)
        {
            var previousVersion = await _context.PagesProcessingStatuses.FirstOrDefaultAsync(c => c.Id == pageNumber);

            if (previousVersion != null)
            {
                switch (state)
                {
                case ProcessingState.SuccessfullProcessed:
                    throw new ArgumentException($"Atempt to save the page {pageNumber} which was already saved before");

                case ProcessingState.SuccessfullReprocessed:
                case ProcessingState.Failed:
                    previousVersion.Status = (int)state;
                    previousVersion.AttemptToProcess++;
                    previousVersion.LastError = errorMessage;
                    break;
                }
            }
            else
            {
                switch (state)
                {
                case ProcessingState.SuccessfullReprocessed:
                    throw new ArgumentException($"Attempt to set success reprocessed state failed. There is no the page {pageNumber} in the unprocessed list");

                case ProcessingState.SuccessfullProcessed:
                case ProcessingState.Failed:
                {
                    var pageProcessingStatus = new PageProcessingStatus();
                    pageProcessingStatus.AttemptToProcess = 1;
                    pageProcessingStatus.Id        = pageNumber;
                    pageProcessingStatus.Status    = (int)state;
                    pageProcessingStatus.LastError = errorMessage;
                    _context.PagesProcessingStatuses.Add(pageProcessingStatus);
                }
                break;
                }
            }

            await _context.SaveChangesAsync();
        }
コード例 #2
0
ファイル: Scraper.xaml.cs プロジェクト: merkator2160/Study
        private void ScrampingStep()
        {
            if (_delayCounter <= 0)
            {
                #region Started

                if (_currentState == States.Started)
                {
                    LogRtbAppendText("Navigating site: " + (_currentVacancyIndex + 1) + " of " + _urlsForVacancyLinkSearch.Count + "\r\n");
                    Navigate(_urlsForVacancyLinkSearch[_currentUrlIndexForLinkSearch++]);
                    _pageStatus = PageProcessingStatus.Collecting;
                    SetCurrentState(States.CollectingLinks);
                    return;
                }

                #endregion

                if (_pageLouded)
                {
                    #region WatingForBrowser

                    if (_currentState == States.WatingForBrowser)
                    {
                        SetCurrentState(_previousState);
                    }

                    #endregion
                    #region CollectingLinks

                    if (_currentState == States.CollectingLinks)
                    {
                        if (_currentUrlIndexForLinkSearch <= _urlsForVacancyLinkSearch.Count)
                        {
                            if (_pageStatus == PageProcessingStatus.Navigating)
                            {
                                LogRtbAppendText("Navigating site: " + (_currentUrlIndexForLinkSearch) + " of " + _urlsForVacancyLinkSearch.Count + "\r\n");
                                Navigate(_urlsForVacancyLinkSearch[_currentUrlIndexForLinkSearch - 1]);
                                _pageStatus = PageProcessingStatus.Collecting;
                            }
                            else
                            {
                                LogRtbAppendText("Processing site: " + (_currentUrlIndexForLinkSearch) + " of " + _urlsForVacancyLinkSearch.Count + "\r\n");

                                GetVacancyes();
                                _currentUrlIndexForLinkSearch++;

                                _delayCounter = GenerateDelay(_delayPodium, _delayDeviation);
                                TimeToNextVacancyLblUpdateContent();

                                _pageStatus = PageProcessingStatus.Navigating;
                            }
                            return;
                        }

                        if (_vacancy.Count == 0)
                        {
                            StopJob("There is no vacancyes.");
                        }

                        PrintCollectedVacancyesReport();
                        SetCurrentState(States.CollectingData);

                        LogRtbAppendText("Navigating site: " + (_currentVacancyIndex + 1) + " of " + _vacancy.Count + "\r\n");
                        Navigate(_vacancy[_currentVacancyIndex].VacancyPageUrl);
                        _pageStatus = PageProcessingStatus.Collecting;
                        return;
                    }

                    #endregion
                    #region CollectingData

                    if (_currentState == States.CollectingData)
                    {
                        if (_currentVacancyIndex < _vacancy.Count)  //_vacancy.Count
                        {
                            if (_pageStatus == PageProcessingStatus.Navigating)
                            {
                                LogRtbAppendText("Navigating site: " + (_currentVacancyIndex + 1) + " of " + _vacancy.Count + "\r\n");
                                Navigate(_vacancy[_currentVacancyIndex].VacancyPageUrl);
                                _pageStatus = PageProcessingStatus.Collecting;
                            }
                            else
                            {
                                LogRtbAppendText("Processing site: " + (_currentVacancyIndex + 1) + " of " + _vacancy.Count + "\r\n");
                                GetVacancyData();
                                WriteVacancyHtmlToFile();
                                SaveVacancyToDb();
                                _currentVacancyIndex++;

                                _delayCounter = GenerateDelay(_delayPodium, _delayDeviation);
                                TimeToNextVacancyLblUpdateContent();

                                _pageStatus = PageProcessingStatus.Navigating;
                            }
                            return;
                        }

                        PrintCollectedDataReport();
                        //WriteAllVacancyesHtmlToFile();
                        //SaveAllVacancyesToDb();
                        StopJob("Job done!");
                    }
                    #endregion
                }
                else
                {
                    #region SET VAITING_FOR_BROWSER_DELAY

                    _delayCounter = VaitingForBrowserDelay;
                    TimeToNextVacancyLblUpdateContent();

                    if (_currentState != States.WatingForBrowser)
                    {
                        _previousState = _currentState;
                    }

                    SetCurrentState(States.WatingForBrowser);

                    #endregion
                }
            }
            else
            {
                _delayCounter--;
                TimeToNextVacancyLblUpdateContent();
            }
        }