private async Task CrawlDescriptor(
List <CrawlerDescriptor> descriptors,
CrawlerSet crawlingSet,
VolatileParametersBase volatileParameters,
CancellationToken feedCtsToken)
{
foreach (var descriptor in descriptors)
{
var semaphore = _domainSemaphores[descriptor.CrawlerDomain];
try
{
await semaphore.WaitAsync(feedCtsToken);
var crawler = _crawlerManager.GetCrawler(descriptor.CrawlerDomain);
var result = await Task.Run(async() =>
await crawler.Crawl(
new CrawlerParameters(
descriptor.CrawlerSourceParameters,
volatileParameters), feedCtsToken), feedCtsToken);
if (result.Success)
{
NewCrawlerBatch?.Invoke(this, new FeedBatch
{
CrawlerResult = result,
SetOfOrigin = crawlingSet
});
}
if (result.HasMore)
{
var task = CrawlDescriptor(new List <CrawlerDescriptor>
{
descriptor
}, crawlingSet, new VolatileParametersBase
{
Page = volatileParameters.Page + 1,
UseCache = volatileParameters.UseCache
}, feedCtsToken);
_branchedTasks.Add(task);
#pragma warning disable 4014
Task.Run(async() => await task, feedCtsToken);
#pragma warning restore 4014
}
}
catch (TaskCanceledException)
{
_logger.LogInformation("Cancelled descriptor crawling.");
}
finally
{
semaphore.Release();
}
}
}
public void WithHistory(List <HistoryFeedEntry> feedHistory, CrawlerSet setOfOrigin, DateTime now)
{
SetOfOrigin = setOfOrigin;
if (feedHistory == null)
{
LastChanged = now;
return;
}
var historyEntry = feedHistory.FirstOrDefault(entry => entry.InternalId == BackingModel.InternalId);
if (historyEntry == null)
{
IsNew = true;
LastChanged = now;
PreviousPrice = BackingModel.Price;
}
else
{
float priceToCompare;
if (Math.Abs(BackingModel.Price - historyEntry.LatestPrice) < 0.001)
{
LastChanged = historyEntry.LastChanged;
PreviousPrice = historyEntry.PreviousPrice;
priceToCompare = historyEntry.PreviousPrice;
}
else
{
LastChanged = now;
PreviousPrice = historyEntry.LatestPrice;
priceToCompare = historyEntry.LatestPrice;
}
if (BackingModel.Price > priceToCompare)
{
PriceChange = PriceChange.Increase;
}
else if (BackingModel.Price < priceToCompare)
{
PriceChange = PriceChange.Decrease;
}
else
{
PriceChange = PriceChange.Stale;
}
if (PriceChange != PriceChange.Stale)
{
PriceDifference = BackingModel.Price - priceToCompare;
}
}
}
public async Task <List <HistoryFeedEntry> > GetHistory(CrawlerSet set)
{
if (_historyDictionary.TryGetValue(set.Guid, out var history))
{
return(history);
}
history = await _dataCache.RetrieveData <List <HistoryFeedEntry> >($"{HistoryDirectory}/{set.Guid}");
if (history != null)
{
_historyDictionary[set.Guid] = history;
}
return(history);
}
public async Task <bool> HasAnyChanged(CrawlerSet setOfOrigin, IEnumerable <ICrawlerResultItem> resultItems)
{
var history = await GetHistory(setOfOrigin);
foreach (var item in resultItems)
{
var historyEntry = history.FirstOrDefault(entry => entry.InternalId.Equals(item.InternalId));
if (historyEntry != null)
{
// price changed
return(Math.Abs(historyEntry.LatestPrice - item.Price) > 0.001);
}
else
{
// new item without history
return(true);
}
}
return(false);
}
public async Task UpdateHistory(CrawlerSet set, List <HistoryFeedEntry> history)
{
_historyDictionary[set.Guid] = history;
await _dataCache.SaveDataAsync($"{HistoryDirectory}/{set.Guid}", history);
}
public CrawlerSetModifiedMessage(CrawlerSet set)
{
ModifiedCrawlerSet = set;
}
public async Task UpdateSet(CrawlerSet set)
{
await _appVariables.CrawlerSets.SetAsync(_sets.ToList());
}
public async Task RemoveSet(CrawlerSet set)
{
_sets.Remove(set);
await _appVariables.CrawlerSets.SetAsync(_sets.ToList());
}
public async Task AddNewSet(CrawlerSet set)
{
_sets.Add(set);
await _appVariables.CrawlerSets.SetAsync(_sets.ToList());
}
public CrawlerSetDetailsPageNavArgs(CrawlerSet set)
{
CrawlerSet = set;
}
