protected List<Ad> CheckForNewAds(Action<CheckForNewAdsState> stateChangedCallback, Action<List<Ad>> completedCallback, CancellationToken cancelationToken)
{
var state = new CheckForNewAdsState();
var result = new List<Ad>();
int totalProcessed = 0;
int maxAds = Managers.SettingsManager.GetSettings().CheckForNewAdsMaxAdsCount;
int frameSize = 100;
int minAds = 200;
double minNewAdsInFrame = 0.1;
foreach (var connector in Managers.ConnectorsManager.GetConnectors())
{
Managers.LogEntriesManager.AddItem(SeverityLevel.Information, string.Format("Starting download from {0}.", connector.Id));
int adsCount = 0;
List<Ad> connectorResult = new List<Ad>();
List<Ad> lastAds = Repositories.AdsRepository.GetLastAds(connector.Id, maxAds);
Ad lastAd = lastAds.FirstOrDefault();
DateTime lastCollectionDate = lastAd == null ? DateTime.Now.Date : lastAd.CollectDate.Date;
Queue<bool> frame = new Queue<bool>(frameSize);/*true for new or republished ad, otherwise false*/
try
{
foreach (var ad in connector.GetAds())
{
adsCount++;
totalProcessed++;
var isNewAd = IsNewOrRepublishedAd(ad, lastAds);
state.Progress = totalProcessed;
state.SourceUrl = connector.Id;
state.Canceled = cancelationToken.IsCancellationRequested;
state.Description = "Processing...";
stateChangedCallback(state);
connectorResult.Add(ad);
frame.Enqueue(isNewAd);
if (frame.Count > frameSize)
{
frame.Dequeue();
}
if ((adsCount > minAds && frame.Count == frameSize && ((double)frame.Where(item => item).Count()/(double)frameSize) < minNewAdsInFrame) ||
adsCount >= maxAds || cancelationToken.IsCancellationRequested)
{
break;
}
}
ProcessCollectDate(connectorResult, DateTime.Now);
state.Description = "Analyzing...";
stateChangedCallback(state);
var acceptance = AddNewOrCreateHistory(connectorResult);
Managers.LogEntriesManager.AddItem(SeverityLevel.Information,
string.Format("{0} Collection from {1} finished. Processed: {2}; New ads: {3}; Added to History: {4}; Rejected: {5};",
this.GetType().Name, connector.Id,
acceptance.Count(),
acceptance.Where(a => a == AdAcceptance.Accepted).Count(),
acceptance.Where(a => a == AdAcceptance.History).Count(),
acceptance.Where(a => a == AdAcceptance.Rejected).Count()));
result.AddRange(connectorResult);
}
catch (Exception ex)
{
Managers.LogEntriesManager.AddItem(SeverityLevel.Error,
string.Format("{0} Download error. {1}", this.GetType().Name, ex.Message), ex.StackTrace);
}
if (cancelationToken.IsCancellationRequested)
{
break;
}
}
completedCallback(result);
return result;
}
protected void OnCheckForNewAdsStateChanged(CheckForNewAdsState state)
{
if (CheckForNewAdsStateChanged != null)
{
CheckForNewAdsStateChanged(this, new EventArgs<CheckForNewAdsState>(state));
}
}