private async Task <ICollection <Show> > Grab(int pageNum) { var shows = new ConcurrentBag <Show>(); var sync = new SemaphoreSlim(5, 5); var tasks = new List <Task>(); var tvMazeShows = await _tvMazeApi.GetShows(pageNum); foreach (var tvMazeShow in tvMazeShows) { await sync.WaitAsync(); tasks.Add( AttemptExecution(async() => { var tvMazeCasts = await _tvMazeApi.GetCasts(tvMazeShow.Id); var show = Mapper.Map <Show>(tvMazeShow); show.People = Mapper.Map <ICollection <Person> >(tvMazeCasts); shows.Add(show); }, 3, 9000) .ContinueWith(t => { sync.Release(); }) ); } await Task.WhenAll(tasks); return(shows.ToArray()); }
public virtual async Task ScrapeData() { Console.WriteLine("Started TvMazeScrape job"); var latestShowId = ShowsRepository.GetLatestAddedShowId(); int nextPage = 0; if (latestShowId > 0) { decimal pageNr = latestShowId / 250; nextPage = (int)Math.Floor(pageNr); } var shows = await TvMazeApi.GetShows(nextPage); Console.WriteLine("Loaded 250 shows"); foreach (var show in shows) { var existingShow = DbContext.Shows.Where(s => s.ShowId.Equals(show.Id)).FirstOrDefault(); if (existingShow != null) { continue; } var castResponse = await TvMazeApi.GetCastMembers((int)show.Id); Console.WriteLine("Received cast for show"); var newShow = new Show() { ShowId = (int)show.Id, Name = show.Name }; await ShowsRepository.AddShow(newShow); HashSet <long> AddedPersons = new HashSet <long>(); foreach (var castMember in castResponse) { // Some shows contain dubplicates if (AddedPersons.Contains(castMember.Person.Id)) { continue; } var person = PersonRepository.CreateOrGet(new Person() { PersonId = (int)castMember.Person.Id, Name = castMember.Person.Name, Birthday = castMember.Person.Birthday?.DateTime ?? new DateTime(1970, 1, 1) }); var showPerson = new ShowPerson() { PersonId = person.PersonId, ShowId = newShow.ShowId }; Console.WriteLine($"ShowId {newShow.ShowId}, PersonId {person.PersonId}"); DbContext.ShowPersons.Add(showPerson); AddedPersons.Add(castMember.Person.Id); } DbContext.SaveChanges(); Console.WriteLine($"Added show {newShow.Name}"); System.Threading.Thread.Sleep(1000); } }