private void PhotosSearchReportProgress(ParallelOperationData <PhotoSearchPageResult> data)
{
if (data.IsCompleted)
{
// finani hlaseni, dopoctu i pocet photo instanci na flickru, ktere maji FLUP tag
var withFlupTag = data.Results.SelectMany(p => p.PhotoCollection).Count(p => p.Tags.Any(t => string.Compare(t, File4Flickr.FLUP_TAG, ignoreCase: true) == 0));
var message = string.Format("Found: {0} flickr files ({1} with FLUP tag).", data.Results?.SelectMany(p => p.PhotoCollection).Count(), withFlupTag);
_logger.Write(message);
}
else
{
// prubezny progress
var message = string.Format("Found: {0} flickr files. (T:{1})", data.Results?.SelectMany(p => p.PhotoCollection).Count(), data.Tasks?.Count);
WriteProgressMsg(data.IsCompleted, message);
}
}
/// <summary>
/// paralelni dohledani seznamu PhotoSetu
/// </summary>
public async Task <List <Photoset> > ParallelPhotosetsGetList(Action <ParallelOperationData <PhotosetSearchPageResult> > progressEvent = null,
int taskCount = DefaultParallelConnections)
{
var pageCount = taskCount;
// pripadna podporapredcasneho ukonceni tasku, viz: https://johnbadams.wordpress.com/2012/03/10/understanding-cancellationtokensource-with-tasks/
// seznam tasku s http dotazy - zatim rozbehnu nekolik uvodnich dotazu a postupne po jejich jednotlivem splneni pridavam dalsi
var taskList = Enumerable.Range(1, pageCount)
.Select(p => PhotosetsGetListForPage(p))
.ToList();
var data = new ParallelOperationData <PhotosetSearchPageResult>(GetTaskSheduler(), progressEvent, taskList);
data.ReportProgress();
while (taskList.Any())
{
// pockam na prvni dokonceny task
Task <PhotosetSearchPageResult> firstFinishedTask = await Task.WhenAny(taskList);
taskList.Remove(firstFinishedTask);
PhotosetSearchPageResult finishedTask = await firstFinishedTask; // vysledek hotoveho tasku
// seznam j*z stazenych photosetu
var downloaded = data.Results.SelectMany(p => p.PhotosetCollection).ToList();
// detekce posledni stranky - downloaduju stale dalsi a dalsi stranky az do doby kdy zacnou prichazet duplicitni zaznamy v PhotoCollection,
// to urcite neni optimalni, ale flickr bohuzel nijak posledni stranku neidentifikuje :-/ takze ji musim takto pseudo dohledat
var anyNewPhotosets = finishedTask.PhotosetCollection.Any(p => !downloaded.Any(r => r.PhotosetId == p.PhotosetId));
// posledni stranka nebyla dohledana, takze rozbehnu task pro stahovani dalsi
if (anyNewPhotosets)
{
data.AddResult(finishedTask);
data.ReportProgress(finishedTask);
pageCount++;
taskList.Add(PhotosetsGetListForPage(pageCount));
}
}
data.SetCompletedState();
data.ReportProgress();
var final = data.Results.SelectMany(p => p.PhotosetCollection).ToList();
return(final);
}
/// <summary>
/// dohledani vsech Photo instacni na flickeru pro zalogovaneho usera
/// </summary>
public async Task <List <Photo> > ParallelGetPhotos(Action <ParallelOperationData <PhotoSearchPageResult> > progressEvent = null,
int taskCount = DefaultParallelConnections)
{
var pageCount = taskCount;
// seznam tasku s http dotazy - zatim rozbehnu nekolik uvodnich dotazu a postupne po jejich jednotlivem splneni pridavam dalsi
var taskList = Enumerable.Range(1, pageCount)
.Select(page => PhotosSearchPage(page))
.ToList();
// data s vysledky procesu, tasky a vstupnimi daty - pro drzeni mezi vysledku i reportovani
var data = new ParallelOperationData <PhotoSearchPageResult>(GetTaskSheduler(), progressEvent, taskList);
data.ReportProgress();
while (taskList.Any())
{
Task <PhotoSearchPageResult> firstFinishedTask = await Task.WhenAny(taskList);
taskList.Remove(firstFinishedTask);
var finishedTask = await firstFinishedTask;
// seznam j*z stazenych photosetu
var downloadedId = data.Results.SelectMany(p => p.PhotoCollection).Select(p => p.PhotoId).ToList();
// detekce posledni stranky - downloaduju stale dalsi a dalsi stranky az do doby kdy zacnou prichazet duplicitni zaznamy v PhotoCollection,
// to urcite neni optimalni, ale flickr bohuzel nijak posledni stranku neidentifikuje :-/ takze ji musim takto pseudo dohledat
//var newPhotos
var neexistujici = finishedTask.PhotoCollection.Select(p => p).Where(p => !downloadedId.Contains(p.PhotoId)).ToList();
// posledni stranka nebyla dohledana, takze rozbehnu task pro stahovani dalsi
if (neexistujici.Any())
{
data.AddResult(finishedTask);
data.ReportProgress(finishedTask);
pageCount++;
taskList.Add(PhotosSearchPage(pageCount));
}
}
data.SetCompletedState();
data.ReportProgress();
var final = data.Results.SelectMany(p => p.PhotoCollection).ToList();
return(final);
}
private void ParallelPhotosetsGetListProgressEvent(ParallelOperationData <PhotosetSearchPageResult> data)
{
var message = string.Format("Found: {0} photosets.", data.Results.SelectMany(p => p.PhotosetCollection).Count());
WriteProgressMsg(data.IsCompleted, message);
}