コード例 #1
0
ファイル: AdsManager.cs プロジェクト: kuzminyv/ad-collector
        protected List<Ad> CheckForNewAds(Action<CheckForNewAdsState> stateChangedCallback, Action<List<Ad>> completedCallback, CancellationToken cancelationToken)
        {
            var state = new CheckForNewAdsState();
            var result = new List<Ad>();
            int totalProcessed = 0;

            int maxAds = Managers.SettingsManager.GetSettings().CheckForNewAdsMaxAdsCount;
            int frameSize = 100;
            int minAds = 200;
            double minNewAdsInFrame = 0.1;

            foreach (var connector in Managers.ConnectorsManager.GetConnectors())
            {
                Managers.LogEntriesManager.AddItem(SeverityLevel.Information, string.Format("Starting download from {0}.", connector.Id));
                int adsCount = 0;

                List<Ad> connectorResult = new List<Ad>();
                List<Ad> lastAds = Repositories.AdsRepository.GetLastAds(connector.Id, maxAds);
                Ad lastAd = lastAds.FirstOrDefault();
                DateTime lastCollectionDate = lastAd == null ? DateTime.Now.Date : lastAd.CollectDate.Date;

                Queue<bool> frame = new Queue<bool>(frameSize);/*true for new or republished ad, otherwise false*/

                try
                {
                    foreach (var ad in connector.GetAds())
                    {
                        adsCount++;
                        totalProcessed++;
                        var isNewAd = IsNewOrRepublishedAd(ad, lastAds);

                        state.Progress = totalProcessed;
                        state.SourceUrl = connector.Id;
                        state.Canceled = cancelationToken.IsCancellationRequested;
                        state.Description = "Processing...";
                        stateChangedCallback(state);

                        connectorResult.Add(ad);
                        frame.Enqueue(isNewAd);
                        if (frame.Count > frameSize)
                        {
                            frame.Dequeue();
                        }

                        if ((adsCount > minAds && frame.Count == frameSize && ((double)frame.Where(item => item).Count()/(double)frameSize) < minNewAdsInFrame) ||
                            adsCount >= maxAds || cancelationToken.IsCancellationRequested)
                        {
                            break;
                        }
                    }

                    ProcessCollectDate(connectorResult, DateTime.Now);

                    state.Description = "Analyzing...";
                    stateChangedCallback(state);

                    var acceptance = AddNewOrCreateHistory(connectorResult);
                    Managers.LogEntriesManager.AddItem(SeverityLevel.Information,
                        string.Format("{0} Collection from {1} finished. Processed: {2}; New ads: {3}; Added to History: {4}; Rejected: {5};",
                        this.GetType().Name, connector.Id,
                        acceptance.Count(),
                        acceptance.Where(a => a == AdAcceptance.Accepted).Count(),
                        acceptance.Where(a => a == AdAcceptance.History).Count(),
                        acceptance.Where(a => a == AdAcceptance.Rejected).Count()));

                    result.AddRange(connectorResult);
                }
                catch (Exception ex)
                {
                    Managers.LogEntriesManager.AddItem(SeverityLevel.Error,
                        string.Format("{0} Download error. {1}", this.GetType().Name, ex.Message), ex.StackTrace);
                }

                if (cancelationToken.IsCancellationRequested)
                {
                    break;
                }
            }

            completedCallback(result);
            return result;
        }
コード例 #2
0
ファイル: AppContext.cs プロジェクト: kuzminyv/ad-collector
 protected void OnCheckForNewAdsStateChanged(CheckForNewAdsState state)
 {
     if (CheckForNewAdsStateChanged != null)
     {
         CheckForNewAdsStateChanged(this, new EventArgs<CheckForNewAdsState>(state));
     }
 }