Example #1
0
        public async Task <ImportStats> ProcessEras(Artist artist, PerformContext ctx)
        {
            var stats = new ImportStats();

            var order = 0;

            foreach (var era in (await PhishinApiRequest <IDictionary <string, IList <string> > >("eras", ctx)).data)
            {
                var dbEra = existingEras.GetValue(era.Key);

                if (dbEra == null)
                {
                    dbEra = await _eraService.Save(new Era()
                    {
                        artist_id  = artist.id,
                        name       = era.Key,
                        order      = order,
                        updated_at = DateTime.Now
                    });

                    existingEras[dbEra.name] = dbEra;

                    stats.Created++;
                }

                foreach (var year in era.Value)
                {
                    yearToEraMapping[year] = dbEra;
                }

                order++;
            }

            return(stats);
        }
Example #2
0
        public async Task <ImportStats> ProcessSongs(Artist artist, PerformContext ctx)
        {
            var stats = new ImportStats();

            var songsToSave = new List <SetlistSong>();

            foreach (var song in (await PhishinApiRequest <IEnumerable <PhishinSmallSong> >("songs", ctx)).data)
            {
                var dbSong = existingSetlistSongs.GetValue(song.id.ToString());

                // skip aliases for now
                if (dbSong == null && song.alias_for.HasValue == false)
                {
                    songsToSave.Add(new SetlistSong()
                    {
                        updated_at          = song.updated_at,
                        artist_id           = artist.id,
                        name                = song.title,
                        slug                = song.slug,
                        upstream_identifier = song.id.ToString()
                    });
                }
            }

            var newSongs = await _setlistSongService.InsertAll(artist, songsToSave);

            foreach (var s in newSongs)
            {
                existingSetlistSongs[s.upstream_identifier] = s;
            }

            stats.Created += newSongs.Count();

            return(stats);
        }
Example #3
0
        public override async Task <ImportStats> ImportDataForArtist(Artist artist, ArtistUpstreamSource src, PerformContext ctx)
        {
            await PreloadData(artist);

            var stats = new ImportStats();

            ctx?.WriteLine("Processing Eras");
            stats += await ProcessEras(artist, ctx);

            ctx?.WriteLine("Processing Tours");
            stats += await ProcessTours(artist, ctx);

            ctx?.WriteLine("Processing Songs");
            stats += await ProcessSongs(artist, ctx);

            ctx?.WriteLine("Processing Venues");
            stats += await ProcessVenues(artist, ctx);

            ctx?.WriteLine("Processing Shows");
            stats += await ProcessShows(artist, src, ctx);

            ctx?.WriteLine("Rebuilding");
            await RebuildShows(artist);
            await RebuildYears(artist);

            return(stats);
            //return await ProcessIdentifiers(artist, await this.http.GetAsync(SearchUrlForArtist(artist)));
        }
        public override async Task <ImportStats> ImportDataForArtist(Artist artist, ArtistUpstreamSource src, PerformContext ctx)
        {
            uint page  = 1;
            var  stats = new ImportStats();

            await PreloadData(artist);

            ctx?.WriteLine($"Requesting page #{page}");

            while (await ImportPage(artist, stats, ctx, await http.GetAsync(UrlForArtist(src, page))))
            {
                page++;

                await Task.Delay(100);

                ctx?.WriteLine($"Requesting page #{page}");
            }

            ctx?.WriteLine("Updating tour start/end dates");
            await UpdateTourStartEndDates(artist);

            // update shows
            await RebuildShows(artist);

            // update years
            await RebuildYears(artist);

            return(stats);
        }
Example #5
0
        public async Task <ImportStats> Import(Artist artist, Func <ArtistUpstreamSource, bool> filterUpstreamSources, string showIdentifier, PerformContext ctx)
        {
            var stats = new ImportStats();

            var srcs = artist.upstream_sources.ToList();

            ctx?.WriteLine($"Found {srcs.Count} valid importers.");
            var prog = ctx?.WriteProgressBar();

            await srcs.AsyncForEachWithProgress(prog, async item =>
            {
                if (filterUpstreamSources != null && !filterUpstreamSources(item))
                {
                    ctx?.WriteLine($"Skipping (rejected by filter): {item.upstream_source_id}, {item.upstream_identifier}");
                    return;
                }

                ctx?.WriteLine($"Importing with {item.upstream_source_id}, {item.upstream_identifier}");

                if (showIdentifier != null)
                {
                    stats += await item.upstream_source.importer.ImportSpecificShowDataForArtist(artist, item, showIdentifier, ctx);
                }
                else
                {
                    stats += await item.upstream_source.importer.ImportDataForArtist(artist, item, ctx);
                }
            });

            return(stats);
        }
Example #6
0
        public override async Task <ImportStats> ImportDataForArtist(Artist artist, ArtistUpstreamSource src, PerformContext ctx)
        {
            var stats = new ImportStats();

            await PreloadData(artist);

            var contents = await FetchUrl(PanicIndexUrl(), ctx);

            var matches = ShowDirMatcher.Matches(contents);

            ctx?.WriteLine($"Check {matches.Count} subdirectories");
            var prog = ctx?.WriteProgressBar();

            var counter = 1;

            foreach (Match match in ShowDirMatcher.Matches(contents))
            {
                var panicDate      = match.Groups[1].Value;
                var panicRecLetter = match.Groups[2].Value;

                // 27-Jul-2016 19:14
                var panicUpdatedAt = DateTime.ParseExact(match.Groups[3].Value.Trim(), "dd-MMM-yyyy HH:mm", CultureInfo.InvariantCulture);

                await ProcessShow(stats, artist, panicDate, panicRecLetter, panicUpdatedAt, ctx);

                prog.SetValue(100.0 * counter / matches.Count);

                counter++;
            }

            await RebuildShows(artist);
            await RebuildYears(artist);

            return(stats);
        }
        async Task <bool> ImportPage(Artist artist, ImportStats stats, PerformContext ctx, HttpResponseMessage res)
        {
            var body = await res.Content.ReadAsStringAsync();

            var json = JsonConvert.DeserializeObject <IList <PhantasyTourShowListing> >(body);

            var prog = ctx?.WriteProgressBar();

            await json.AsyncForEachWithProgress(prog, async show =>
            {
                if (show.showDate.ToUniversalTime() > DateTime.UtcNow)
                {
                    // future shows can't have recordings
                    return;
                }

                var showId  = UpstreamIdentifierForPhantasyTourId(show.showId);
                var venueId = UpstreamIdentifierForPhantasyTourId(show.venueId);

                // we have no way to tell if things get updated, so just pull once
                if (!existingSetlistShows.ContainsKey(showId))
                {
                    await ImportSingle(artist, stats, ctx, show.showId);
                }
            });

            // once we aren't at the full items per page this is the last page and we should stop
            var shouldContinue = json.Count == ITEMS_PER_PAGE;

            return(shouldContinue);
        }
Example #8
0
        async Task <Tuple <bool, ImportStats> > ProcessSetlistPage(Artist artist, HttpResponseMessage res, PerformContext ctx, IProgressBar prog)
        {
            var body = await res.Content.ReadAsStringAsync();

            Relisten.Vendor.SetlistFm.SetlistsRootObject root = null;
            try
            {
                root = JsonConvert.DeserializeObject <Relisten.Vendor.SetlistFm.SetlistsRootObject>(
                    body,
                    new Vendor.SetlistFm.TolerantListConverter <Vendor.SetlistFm.Song>(),
                    new Vendor.SetlistFm.TolerantListConverter <Vendor.SetlistFm.Set>(),
                    new Vendor.SetlistFm.TolerantSetsConverter()
                    );
            }
            catch (JsonReaderException e)
            {
                ctx?.WriteLine("Failed to parse {0}:\n{1}", res.RequestMessage.RequestUri.ToString(), body);
                throw e;
            }

            var stats = new ImportStats();

            var count = 1;

            foreach (var setlist in root.setlists.setlist)
            {
                if (setlist.sets.set.Count > 0)
                {
                    Stopwatch s = new Stopwatch();
                    s.Start();

                    // ctx?.WriteLine("Indexing setlist: {0}/{1}...", artist.name, setlist.eventDate);

                    try
                    {
                        var thisStats = await ProcessSetlist(artist, setlist);

                        s.Stop();
                        // ctx?.WriteLine("...success in {0}! Stats: {1}", s.Elapsed, thisStats);

                        stats += thisStats;
                    }
                    catch (Exception e)
                    {
                        s.Stop();
                        ctx?.WriteLine("{0}/{1}...failed in {2}! Stats: {3}", artist.name, setlist.eventDate, s.Elapsed, e.Message);

                        throw e;
                    }

                    prog.SetValue(((100.0 * (root.setlists.page - 1) * root.setlists.itemsPerPage) + count * 1.0) / root.setlists.total);

                    count++;
                }
            }

            var hasMorePages = root.setlists.page < Math.Ceiling(1.0 * root.setlists.total / root.setlists.itemsPerPage);

            return(new Tuple <bool, ImportStats>(hasMorePages, stats));
        }
        public override async Task <ImportStats> ImportDataForArtist(Artist artist, ArtistUpstreamSource src, PerformContext ctx)
        {
            var stats = new ImportStats();

            await PreloadData(artist);

            var resp = await http.GetAsync(ShowPagesListingUrl(src));

            var showFilesResponse = await resp.Content.ReadAsStringAsync();

            var showFiles = JsonConvert.DeserializeObject <List <string> >(showFilesResponse);

            var files = showFiles
                        .Select(f =>
            {
                var fileName = Path.GetFileName(f);
                return(new FileMetaObject
                {
                    DisplayDate = fileName.Substring(0, 10),
                    Date = DateTime.Parse(fileName.Substring(0, 10)),
                    FilePath = f,
                    Identifier = fileName.Remove(fileName.LastIndexOf(".html", StringComparison.OrdinalIgnoreCase))
                });
            })
                        .ToList()
            ;

            ctx?.WriteLine($"Checking {files.Count} html files");
            var prog = ctx?.WriteProgressBar();

            await files.AsyncForEachWithProgress(prog, async f =>
            {
                if (existingSetlistShows.ContainsKey(f.Identifier))
                {
                    return;
                }

                var url          = ShowPageUrl(src, f.FilePath);
                var pageResp     = await http.GetAsync(url);
                var pageContents = await pageResp.Content.ReadAsStringAsync();

                await ProcessPage(stats, artist, f, pageContents, pageResp.Content.Headers.LastModified?.UtcDateTime ?? DateTime.UtcNow, ctx);
            });

            if (artist.features.tours)
            {
                await UpdateTourStartEndDates(artist);
            }

            ctx.WriteLine("Rebuilding shows and years");

            // update shows
            await RebuildShows(artist);

            // update years
            await RebuildYears(artist);

            return(stats);
        }
        async Task ImportSingle(Artist artist, ImportStats stats, PerformContext ctx, int showId)
        {
            var policy = Policy
                         .Handle <JsonReaderException>()
                         .WaitAndRetryAsync(5, retryAttempt => TimeSpan.FromSeconds(Math.Pow(2, retryAttempt) / 2.0));

            await policy.ExecuteAsync(() => _ImportSingle(artist, stats, ctx, showId));
        }
Example #11
0
        public async Task <ImportStats> ProcessShows(Artist artist, PerformContext ctx)
        {
            var stats = new ImportStats();

            var shows = (await PhishinApiRequest <IEnumerable <PhishinSmallShow> >("shows", ctx, "date")).ToList();

            var prog = ctx?.WriteProgressBar();

            await shows.AsyncForEachWithProgress(prog, async show =>
            {
                var dbSource = existingSources.GetValue(show.id.ToString());

                if (dbSource == null)
                {
                    dbSource = await ProcessShow(stats, artist, new Source()
                    {
                        updated_at          = show.updated_at,
                        artist_id           = artist.id,
                        venue_id            = existingVenues[show.venue_id.ToString()].id,
                        display_date        = show.date,
                        upstream_identifier = show.id.ToString(),
                        is_soundboard       = show.sbd,
                        is_remaster         = show.remastered,
                        description         = "",
                        taper_notes         = show.taper_notes
                    }, ctx);

                    existingSources[dbSource.upstream_identifier] = dbSource;

                    stats.Created++;
                }
                else if (show.updated_at > dbSource.updated_at)
                {
                    dbSource.updated_at          = show.updated_at;
                    dbSource.venue_id            = existingVenues[show.venue_id.ToString()].id;
                    dbSource.display_date        = show.date;
                    dbSource.upstream_identifier = show.id.ToString();
                    dbSource.is_soundboard       = show.sbd;
                    dbSource.is_remaster         = show.remastered;
                    dbSource.description         = "";
                    dbSource.taper_notes         = show.taper_notes;

                    stats.Removed += await _sourceService.DropAllSetsAndTracksForSource(dbSource);

                    dbSource = await ProcessShow(stats, artist, dbSource, ctx);

                    existingSources[dbSource.upstream_identifier] = dbSource;

                    stats.Updated++;
                }
            });

            return(stats);
        }
Example #12
0
        public override async Task <ImportStats> ImportDataForArtist(Artist artist, ArtistUpstreamSource src, PerformContext ctx)
        {
            var stats = new ImportStats();

            await PreloadData(artist);

            var contents = await FetchUrl(PanicIndexUrl(), ctx);

            var tracks = JsonConvert.DeserializeObject <List <PanicStream.PanicStreamTrack> >(contents);

            var tracksByShow = tracks
                               .Where(t => t.SourceName != null)
                               .GroupBy(t => t.ShowDate)
                               .Select(g => new
            {
                ShowDate = g.Key,
                Sources  = g
                           .GroupBy(subg => subg.SourceName)
                           .Select(subg => new
                {
                    SourceName = subg.Key,
                    Tracks     = subg.ToList()
                })
            })
                               .ToList();

            ctx?.WriteLine($"Found {tracksByShow.Count} shows");

            var prog = ctx?.WriteProgressBar();

            await tracksByShow.AsyncForEachWithProgress(prog, async grp =>
            {
                foreach (var source in grp.Sources)
                {
                    try {
                        await ProcessShow(stats, artist, src, grp.ShowDate, source.SourceName, source.Tracks, ctx);
                    }
                    catch (Exception e) {
                        ctx?.WriteLine("EXCEPTION: " + e.Message);
                        ctx?.WriteLine("Source name: " + source.SourceName);
                        ctx?.WriteLine(e.ToString());
                        ctx?.WriteLine(JsonConvert.SerializeObject(source));
                    }
                }
            });

            ctx?.WriteLine("Rebuilding shows...");
            await RebuildShows(artist);

            ctx?.WriteLine("Rebuilding years...");
            await RebuildYears(artist);

            return(stats);
        }
Example #13
0
        public async Task <ImportStats> ProcessVenues(Artist artist, PerformContext ctx)
        {
            var stats = new ImportStats();

            foreach (var venue in (await PhishinApiRequest <IEnumerable <PhishinSmallVenue> >("venues", ctx)).data)
            {
                var dbVenue = existingVenues.GetValue(venue.id.ToString());

                if (dbVenue == null)
                {
                    var sc = new VenueWithShowCount()
                    {
                        updated_at          = venue.updated_at,
                        artist_id           = artist.id,
                        name                = venue.name,
                        location            = venue.location,
                        slug                = Slugify(venue.name),
                        latitude            = venue.latitude,
                        longitude           = venue.longitude,
                        past_names          = venue.past_names,
                        upstream_identifier = venue.id.ToString()
                    };

                    var createdDb = await _venueService.Save(sc);

                    sc.id = createdDb.id;

                    existingVenues[sc.upstream_identifier] = sc;

                    stats.Created++;

                    dbVenue = sc;
                }
                else if (venue.updated_at > dbVenue.updated_at)
                {
                    dbVenue.name       = venue.name;
                    dbVenue.location   = venue.location;
                    dbVenue.longitude  = venue.longitude;
                    dbVenue.latitude   = venue.latitude;
                    dbVenue.past_names = venue.past_names;
                    dbVenue.updated_at = venue.updated_at;

                    await _venueService.Save(dbVenue);

                    existingVenues[dbVenue.upstream_identifier] = dbVenue;

                    stats.Updated++;
                }
            }

            return(stats);
        }
Example #14
0
        private async Task ProcessSetlistShow(ImportStats stats, PhishinShow show, Artist artist, Source dbSource, IDictionary <string, SourceSet> sets)
        {
            var dbShow = existingSetlistShows.GetValue(show.date);

            var addSongs = false;

            if (dbShow == null)
            {
                dbShow = await _setlistShowService.Save(new SetlistShow()
                {
                    artist_id           = artist.id,
                    upstream_identifier = show.date,
                    date       = DateTime.Parse(show.date),
                    venue_id   = existingVenues[show.venue.id.ToString()].id,
                    tour_id    = existingTours[show.tour_id.ToString()].id,
                    era_id     = yearToEraMapping.GetValue(show.date.Substring(0, 4), yearToEraMapping["1983-1987"]).id,
                    updated_at = dbSource.updated_at
                });

                stats.Created++;

                addSongs = true;
            }
            else if (show.updated_at > dbShow.updated_at)
            {
                dbShow.date       = DateTime.Parse(show.date);
                dbShow.venue_id   = existingVenues[show.venue.id.ToString()].id;
                dbShow.tour_id    = existingTours[show.tour_id.ToString()].id;
                dbShow.era_id     = yearToEraMapping.GetValue(show.date.Substring(0, 4), yearToEraMapping["1983-1987"]).id;
                dbShow.updated_at = dbSource.updated_at;

                dbShow = await _setlistShowService.Save(dbShow);

                stats.Updated++;
                stats.Removed += await _setlistShowService.RemoveSongPlays(dbShow);

                addSongs = true;
            }

            if (addSongs)
            {
                var dbSongs = show.tracks.
                              SelectMany(phishinTrack => phishinTrack.song_ids.Select(song_id => existingSetlistSongs.GetValue(song_id.ToString()))).
                              Where(t => t != null).
                              GroupBy(t => t.upstream_identifier).
                              Select(g => g.First()).
                              ToList()
                ;

                stats.Created += await _setlistShowService.AddSongPlays(dbShow, dbSongs);
            }
        }
Example #15
0
        private async Task <ImportStats> ProcessSource(Artist artist, Source dbSource, PerformContext ctx)
        {
            var stats = new ImportStats();

            var ratings = await ScrapePhishNetForSource(dbSource, ctx);

            var dirty = false;

            if (dbSource.num_ratings != ratings.RatingVotesCast)
            {
                dbSource.num_ratings = ratings.RatingVotesCast;
                dbSource.avg_rating  = ratings.RatingAverage;

                dirty = true;
            }

            if (dbSource.num_reviews != ratings.NumberOfReviewsWritten)
            {
                var reviewsTask = GetPhishNetApiReviews(dbSource, ctx);
                var setlistTask = GetPhishNetApiSetlist(dbSource, ctx);

                await Task.WhenAll(reviewsTask, setlistTask);

                var dbReviews = reviewsTask.Result.Select(rev =>
                {
                    return(new SourceReview()
                    {
                        rating = null,
                        title = null,
                        review = rev.review,
                        author = rev.author,
                        updated_at = DateTimeOffset.FromUnixTimeSeconds(rev.tstamp).UtcDateTime
                    });
                }).ToList();

                dbSource.num_reviews = dbReviews.Count();
                dbSource.description = setlistTask.Result.setlistnotes + "\n\n\n" + setlistTask.Result.setlistdata;

                dirty = true;

                await ReplaceSourceReviews(stats, dbSource, dbReviews);
            }

            if (dirty)
            {
                await _sourceService.Save(dbSource);

                stats.Updated++;
            }

            return(stats);
        }
Example #16
0
        private async Task <IEnumerable <SourceReview> > ReplaceSourceReviews(ImportStats stats, Source source, IEnumerable <SourceReview> reviews)
        {
            foreach (var review in reviews)
            {
                review.source_id = source.id;
            }

            var res = await _sourceReviewService.UpdateAll(source, reviews);

            stats.Created += res.Count();

            return(res);
        }
Example #17
0
        private async Task <Source> ProcessShow(ImportStats stats, Artist artist, Source dbSource, PerformContext ctx)
        {
            var fullShow = await PhishinApiRequest <PhishinShow>("shows/" + dbSource.upstream_identifier, ctx);

            dbSource.has_jamcharts = fullShow.tags.Contains("Jamcharts");
            dbSource = await _sourceService.Save(dbSource);

            var sets = new Dictionary <string, SourceSet>();

            foreach (var track in fullShow.tracks)
            {
                var set = sets.GetValue(track.set);

                if (set == null)
                {
                    set = await _sourceSetService.Insert(new SourceSet()
                    {
                        source_id  = dbSource.id,
                        index      = SetIndexForIdentifier(track.set),
                        name       = track.set_name,
                        is_encore  = track.set[0] == 'E',
                        updated_at = dbSource.updated_at
                    });

                    // this needs to be set after loading from the db
                    set.tracks = new List <SourceTrack>();

                    stats.Created++;

                    sets[track.set] = set;
                }

                set.tracks.Add(new SourceTrack()
                {
                    source_set_id  = set.id,
                    source_id      = dbSource.id,
                    title          = track.title,
                    duration       = track.duration,
                    track_position = track.position,
                    slug           = Slugify(track.title),
                    mp3_url        = track.mp3,
                    updated_at     = dbSource.updated_at
                });
            }

            stats.Created += (await _sourceTrackService.InsertAll(sets.SelectMany(kvp => kvp.Value.tracks))).Count();

            await ProcessSetlistShow(stats, fullShow, artist, dbSource, sets);

            return(dbSource);
        }
Example #18
0
        private async Task <IEnumerable <SourceReview> > ReplaceSourceReviews(ImportStats stats, Source source, IEnumerable <SourceReview> reviews)
        {
            stats.Removed += await _sourceReviewService.RemoveAllForSource(source);

            foreach (var review in reviews)
            {
                review.source_id = source.id;
            }

            var res = await _sourceReviewService.InsertAll(reviews);

            stats.Created += res.Count();

            return(res);
        }
Example #19
0
        public async Task <ImportStats> Import(Artist artist, PerformContext ctx)
        {
            var stats = new ImportStats();

            var srcs = artist.upstream_sources.ToList();

            ctx?.WriteLine($"Found {srcs.Count} valid importers.");
            var prog = ctx?.WriteProgressBar();

            await srcs.AsyncForEachWithProgress(prog, async item =>
            {
                ctx?.WriteLine($"Importing with {item.GetType()}");
                stats += await item.upstream_source.importer.ImportDataForArtist(artist, item, ctx);
            });

            return(stats);
        }
Example #20
0
        private async Task <ImportStats> ProcessIdentifiers(Artist artist, HttpResponseMessage res, PerformContext ctx)
        {
            var stats = new ImportStats();

            var json = await res.Content.ReadAsStringAsync();

            var root = JsonConvert.DeserializeObject <Relisten.Vendor.ArchiveOrg.SearchRootObject>(
                json,
                new Relisten.Vendor.ArchiveOrg.TolerantArchiveDateTimeConverter()
                );

            ctx?.WriteLine($"Checking {root.response.docs.Count} archive.org results");

            var prog = ctx?.WriteProgressBar();

            await root.response.docs.AsyncForEachWithProgress(prog, async doc =>
            {
                var dbShow = existingSources.GetValue(doc.identifier);
                if (dbShow == null ||
                    doc._iguana_updated_at > dbShow.updated_at)
                {
                    ctx?.WriteLine("Pulling https://archive.org/metadata/{0}", doc.identifier);

                    var detailRes   = await http.GetAsync(DetailsUrlForIdentifier(doc.identifier));
                    var detailsJson = await detailRes.Content.ReadAsStringAsync();
                    var detailsRoot = JsonConvert.DeserializeObject <Relisten.Vendor.ArchiveOrg.Metadata.RootObject>(
                        detailsJson,
                        new Vendor.ArchiveOrg.TolerantStringConverter()
                        );

                    stats += await ImportSingleIdentifier(artist, dbShow, doc, detailsRoot, ctx);
                }
            });

            // update shows
            await RebuildShows(artist);

            // update years
            await RebuildYears(artist);

            return(stats);
        }
Example #21
0
        public async Task <ImportStats> ProcessTours(Artist artist, PerformContext ctx)
        {
            var stats = new ImportStats();

            foreach (var tour in (await PhishinApiRequest <IEnumerable <PhishinSmallTour> >("tours", ctx)).data)
            {
                var dbTour = existingTours.GetValue(tour.id.ToString());

                if (dbTour == null)
                {
                    dbTour = await _tourService.Save(new Tour
                    {
                        updated_at          = tour.updated_at,
                        artist_id           = artist.id,
                        start_date          = DateTime.Parse(tour.starts_on),
                        end_date            = DateTime.Parse(tour.ends_on),
                        name                = tour.name,
                        slug                = Slugify(tour.name),
                        upstream_identifier = tour.id.ToString()
                    });

                    existingTours[dbTour.upstream_identifier] = dbTour;

                    stats.Created++;
                }
                else if (tour.updated_at > dbTour.updated_at)
                {
                    dbTour.start_date = DateTime.Parse(tour.starts_on);
                    dbTour.end_date   = DateTime.Parse(tour.ends_on);
                    dbTour.name       = tour.name;

                    dbTour = await _tourService.Save(dbTour);

                    existingTours[dbTour.upstream_identifier] = dbTour;

                    stats.Updated++;
                }
            }

            return(stats);
        }
        async Task <bool> ImportPage(Artist artist, ImportStats stats, PerformContext ctx, HttpResponseMessage res)
        {
            var body = await res.Content.ReadAsStringAsync();

            var json = JsonConvert.DeserializeObject <IList <PhantasyTourShowListing> >(body);

            if (json == null)
            {
                ctx?.WriteLine("Improper response from phantasytour.com: " + body);
                ctx?.WriteLine($"Status code: {res.StatusCode}. Headers: {string.Join("\n", res.Headers.Select(h => h.Key + ": " + string.Join(" || ", h.Value)))}");
                ctx?.WriteLine($"Request url: {res.RequestMessage.RequestUri}. Headers: {string.Join("\n", res.RequestMessage.Headers.Select(h => h.Key + ": " + string.Join(" || ", h.Value)))}");
                return(false);
            }

            var prog = ctx?.WriteProgressBar();

            await json.AsyncForEachWithProgress(prog, async show =>
            {
                if (show.dateTime.ToUniversalTime() > DateTime.UtcNow)
                {
                    // future shows can't have recordings
                    return;
                }

                var showId  = UpstreamIdentifierForPhantasyTourId(show.id);
                var venueId = UpstreamIdentifierForPhantasyTourId(show.venue.id);

                // we have no way to tell if things get updated, so just pull once
                if (!existingSetlistShows.ContainsKey(showId))
                {
                    await ImportSingle(artist, stats, ctx, show.id);
                }
            });

            // once we aren't at the full items per page this is the last page and we should stop
            var shouldContinue = json.Count == ITEMS_PER_PAGE;

            return(shouldContinue);
        }
Example #23
0
        public override async Task <ImportStats> ImportDataForArtist(Artist artist, ArtistUpstreamSource src, PerformContext ctx)
        {
            var stats = new ImportStats();

            var shows = (await _sourceService.AllForArtist(artist)).OrderBy(s => s.display_date).ToList();

            var prog = ctx?.WriteProgressBar();

            ctx?.WriteLine($"Processing {shows.Count} shows");

            await shows.ForEachAsync(async dbSource =>
            {
                stats += await ProcessSource(artist, src, dbSource, ctx);
            }, prog, 1);

            ctx?.WriteLine("Rebuilding...");

            await RebuildShows(artist);
            await RebuildYears(artist);

            return(stats);
        }
Example #24
0
        public override async Task <ImportStats> ImportDataForArtist(Artist artist, ArtistUpstreamSource src, PerformContext ctx)
        {
            var stats = new ImportStats();

            await PreloadData(artist);

            var files = Directory.EnumerateFiles(ShowFilesDirectory)
                        .Where(f => f.EndsWith(".html", StringComparison.OrdinalIgnoreCase))
                        .Select(f =>
            {
                var fileName = Path.GetFileName(f);
                return(new FileMetaObject
                {
                    DisplayDate = fileName.Substring(0, 10),
                    Date = DateTime.Parse(fileName.Substring(0, 10)),
                    FilePath = f,
                    Identifier = fileName.Remove(fileName.LastIndexOf(".html", StringComparison.OrdinalIgnoreCase))
                });
            })
                        .ToList()
            ;

            ctx?.WriteLine($"Checking {files.Count} html files");
            var prog = ctx?.WriteProgressBar();


            await files.AsyncForEachWithProgress(prog, async f =>
            {
                await ProcessPage(stats, artist, f, File.ReadAllText(f.FilePath), ctx);
            });

            if (artist.features.tours)
            {
                await UpdateTourStartEndDates(artist);
            }

            return(stats);
        }
Example #25
0
        private async Task ProcessShow(ImportStats stats, Artist artist, ArtistUpstreamSource upstreamSrc, string showDate, string sourceName, IList <PanicStream.PanicStreamTrack> sourceTracks, PerformContext ctx)
        {
            var upstreamId = sourceName;
            var dbSource   = existingSources.GetValue(upstreamId);

            var panicUpdatedAt = sourceTracks
                                 .Where(t => t.System.ParsedModificationTime.HasValue)
                                 .Max(t => t.System.ParsedModificationTime.Value);

            if (dbSource != null && dbSource.updated_at <= panicUpdatedAt)
            {
                return;
            }

            var isUpdate = dbSource != null;

            var src = new Source
            {
                artist_id           = artist.id,
                display_date        = showDate,
                is_soundboard       = false,
                is_remaster         = false,
                has_jamcharts       = false,
                avg_rating          = 0,
                num_reviews         = 0,
                avg_rating_weighted = 0,
                upstream_identifier = upstreamId,
                taper_notes         = "",
                updated_at          = panicUpdatedAt
            };

            if (isUpdate)
            {
                src.id = dbSource.id;
            }

            dbSource = await _sourceService.Save(src);

            existingSources[dbSource.upstream_identifier] = dbSource;

            if (isUpdate)
            {
                stats.Updated++;
            }
            else
            {
                stats.Created++;
                stats.Created += (await linkService.AddLinksForSource(dbSource, new[]
                {
                    new Link
                    {
                        source_id = dbSource.id,
                        for_ratings = false,
                        for_source = true,
                        for_reviews = false,
                        upstream_source_id = upstreamSrc.upstream_source_id,
                        url = $"https://www.panicstream.com/vault/widespread-panic/{dbSource.display_date.Substring(0, 4)}-streams/",
                        label = "View show page on panicstream.com"
                    }
                })).Count();
            }

            var dbSet = await _sourceSetService.Update(dbSource, new SourceSet
            {
                source_id  = dbSource.id,
                index      = 0,
                is_encore  = false,
                name       = "Default Set",
                updated_at = panicUpdatedAt
            });

            stats.Created++;

            var trackIndex = 0;
            var mp3s       = sourceTracks
                             .OrderBy(t => t.FileName)
                             .Select(t =>
            {
                var trackName = t.FileName
                                .Replace(".mp3", "")
                                .Replace(".MP3", "")
                                .Replace(".M4A", "")
                                .Replace(".m4a", "")
                                .Trim();

                var cleanedTrackName = Regex.Replace(trackName, @"(wsp[0-9-]+d\d+t\d+\.)|(^\d+ ?-? ?)", "").Trim();

                if (cleanedTrackName.Length != 0)
                {
                    trackName = cleanedTrackName;
                }

                trackIndex++;

                return(new SourceTrack
                {
                    source_id = dbSource.id,
                    source_set_id = dbSet.id,
                    track_position = trackIndex,
                    duration = ((int?)t.Composite?.CalculatedDuration.TotalSeconds ?? (int?)0).Value,
                    title = trackName,
                    slug = SlugifyTrack(trackName),
                    mp3_url = t.AbsoluteUrl(_configuration["PANIC_KEY"]),
                    updated_at = panicUpdatedAt,
                    artist_id = artist.id
                });
            });

            ResetTrackSlugCounts();

            await _sourceTrackService.InsertAll(dbSource, mp3s);

            stats.Created += mp3s.Count();
        }
        async Task _ImportSingle(Artist artist, ImportStats stats, PerformContext ctx, int showId)
        {
            ctx?.WriteLine($"Requesting page for show id {showId}");

            var res = await http.GetAsync(UrlForShow(showId));

            var body = await res.Content.ReadAsStringAsync();

            ctx?.WriteLine($"Result: [{res.StatusCode}]: {body.Length}");

            var json = JsonConvert.DeserializeObject <PhantasyTourEnvelope>(body).data;

            var now = DateTime.UtcNow;

            Venue dbVenue = existingVenues.GetValue(UpstreamIdentifierForPhantasyTourId(json.venue.id));

            if (dbVenue == null)
            {
                var sc = new VenueWithShowCount()
                {
                    updated_at          = now,
                    artist_id           = artist.id,
                    name                = json.venue.name,
                    latitude            = null,
                    longitude           = null,
                    location            = $"{json.venue.city}, {json.venue.state}, {json.venue.country}",
                    upstream_identifier = UpstreamIdentifierForPhantasyTourId(json.venue.id),
                    slug                = Slugify($"{json.venue.name}, {json.venue.city}, {json.venue.state}")
                };

                dbVenue = await _venueService.Save(sc);

                sc.id = dbVenue.id;

                existingVenues[dbVenue.upstream_identifier] = sc;

                stats.Created++;
            }

            ctx?.WriteLine($"Importing show id {showId}");

            // tour
            Tour dbTour = null;

            if (artist.features.tours)
            {
                var tour_name     = json.tour?.name ?? "Not Part of a Tour";
                var tour_upstream = UpstreamIdentifierForPhantasyTourId(json.tour?.id ?? -1);

                dbTour = existingTours.GetValue(tour_upstream);
                if (dbTour == null)
                {
                    dbTour = await _tourService.Save(new Tour()
                    {
                        updated_at          = now,
                        artist_id           = artist.id,
                        start_date          = null,
                        end_date            = null,
                        name                = tour_name,
                        slug                = Slugify(tour_name),
                        upstream_identifier = tour_upstream
                    });

                    existingTours[dbTour.upstream_identifier] = dbTour;

                    stats.Created++;
                }
            }

            var dbShow = existingSetlistShows.GetValue(UpstreamIdentifierForPhantasyTourId(json.id));
            var date   = json.dateTimeUtc.Date;

            if (dbShow == null)
            {
                dbShow = await _setlistShowService.Save(new SetlistShow()
                {
                    artist_id           = artist.id,
                    updated_at          = now,
                    date                = date,
                    upstream_identifier = UpstreamIdentifierForPhantasyTourId(json.id),
                    venue_id            = dbVenue.id,
                    tour_id             = artist.features.tours ? dbTour?.id : null
                });

                existingSetlistShows[dbShow.upstream_identifier] = dbShow;

                stats.Created++;
            }

            // phantasy tour doesn't provide much info about tours so we need to find the start
            // and end date ourselves.
            if (artist.features.tours &&
                (dbTour.start_date == null ||
                 dbTour.end_date == null ||
                 dbShow.date < dbTour.start_date ||
                 dbShow.date > dbTour.end_date))
            {
                if (!tourToStartDate.ContainsKey(dbTour.upstream_identifier) ||
                    dbShow.date < tourToStartDate[dbTour.upstream_identifier])
                {
                    tourToStartDate[dbTour.upstream_identifier] = dbShow.date;
                }

                if (!tourToEndDate.ContainsKey(dbTour.upstream_identifier) ||
                    dbShow.date > tourToEndDate[dbTour.upstream_identifier])
                {
                    tourToEndDate[dbTour.upstream_identifier] = dbShow.date;
                }
            }

            var songs = json.sets.
                        SelectMany(set => set.songs).
                        Select(song => new { Name = song.name, Slug = Slugify(song.name) }).
                        GroupBy(song => song.Slug).
                        Select(grp => grp.First()).
                        ToList();

            var dbSongs = existingSetlistSongs.
                          Where(kvp => songs.Select(song => song.Slug).Contains(kvp.Key)).
                          Select(kvp => kvp.Value).
                          ToList();

            if (songs.Count != dbSongs.Count)
            {
                var newSongs = songs.
                               Where(song => dbSongs.Find(dbSong => dbSong.slug == song.Slug) == null).
                               Select(song => new SetlistSong()
                {
                    artist_id           = artist.id,
                    name                = song.Name,
                    slug                = song.Slug,
                    updated_at          = now,
                    upstream_identifier = song.Slug
                }).
                               ToList();

                var justAdded = await _setlistSongService.InsertAll(artist, newSongs);

                dbSongs.AddRange(justAdded);
                stats.Created += newSongs.Count;

                foreach (var justAddedSong in justAdded)
                {
                    existingSetlistSongs[justAddedSong.upstream_identifier] = justAddedSong;
                }
            }

            stats += await _setlistShowService.UpdateSongPlays(dbShow, dbSongs);
        }
Example #27
0
        private async Task ProcessPage(ImportStats stats, Artist artist, FileMetaObject meta, string pageContents, PerformContext ctx)
        {
            var dbShow = existingSetlistShows.GetValue(meta.Identifier);

            if (dbShow != null)
            {
                return;
            }

            var html = new HtmlDocument();

            html.LoadHtml(pageContents);

            var root = html.DocumentNode;

            var ps = root.DescendantsWithClass("venue-name").ToList();

            var bandName = ps[1].InnerText.CollapseSpacesAndTrim();

            var venueName            = ps[0].InnerText.CollapseSpacesAndTrim();
            var venueCityOrCityState = root.DescendantsWithClass("venue-address").Single().InnerText.CollapseSpacesAndTrim().TrimEnd(',');
            var venueCountry         = root.DescendantsWithClass("venue-country").Single().InnerText.CollapseSpacesAndTrim();

            string tourName = ps.Count > 2 ? ps[2].InnerText.CollapseSpacesAndTrim() : "Not Part of a Tour";

            var venueUpstreamId = "jerrygarcia.com_" + venueName;

            var dbVenue = existingVenues.GetValue(venueUpstreamId);

            if (dbVenue == null)
            {
                dbVenue = await _venueService.Save(new Venue
                {
                    artist_id           = artist.id,
                    name                = venueName,
                    location            = venueCityOrCityState + ", " + venueCountry,
                    upstream_identifier = venueUpstreamId,
                    slug                = Slugify(venueName)
                });

                existingVenues[dbVenue.upstream_identifier] = dbVenue;

                stats.Created++;
            }

            var dbTour = existingTours.GetValue(tourName);

            if (dbTour == null && artist.features.tours)
            {
                dbTour = await _tourService.Save(new Tour
                {
                    artist_id           = artist.id,
                    name                = tourName,
                    slug                = Slugify(tourName),
                    upstream_identifier = tourName
                });

                existingTours[dbTour.upstream_identifier] = dbTour;

                stats.Created++;
            }

            dbShow = await _setlistShowService.Save(new SetlistShow
            {
                artist_id           = artist.id,
                tour_id             = dbTour?.id,
                venue_id            = dbVenue.id,
                date                = meta.Date,
                upstream_identifier = meta.Identifier
            });

            existingSetlistShows[dbShow.upstream_identifier] = dbShow;

            stats.Created++;

            var dbSongs = root.Descendants("ol")
                          .SelectMany(node => node.Descendants("li"))
                          .Select(node =>
            {
                var trackName = node.InnerText.Trim().TrimEnd('>', '*', ' ');
                var slug      = Slugify(trackName);

                return(new SetlistSong
                {
                    artist_id = artist.id,
                    name = trackName,
                    slug = slug,
                    upstream_identifier = slug
                });
            })
                          .GroupBy(s => s.upstream_identifier)
                          .Select(g => g.First())
                          .ToList()
            ;

            var dbSongsToAdd = dbSongs.Where(song => !existingSetlistSongs.ContainsKey(song.upstream_identifier));

            dbSongs = dbSongs.Where(song => existingSetlistSongs.ContainsKey(song.upstream_identifier))
                      .Select(song => existingSetlistSongs[song.upstream_identifier])
                      .ToList()
            ;

            var added = await _setlistSongService.InsertAll(artist, dbSongsToAdd);

            foreach (var s in added)
            {
                existingSetlistSongs[s.upstream_identifier] = s;
            }

            stats.Created += added.Count();
            dbSongs.AddRange(added);

            if (artist.features.tours &&
                (dbTour.start_date == null ||
                 dbTour.end_date == null ||
                 dbShow.date < dbTour.start_date ||
                 dbShow.date > dbTour.end_date))
            {
                if (!tourToStartDate.ContainsKey(dbTour.upstream_identifier) ||
                    dbShow.date < tourToStartDate[dbTour.upstream_identifier])
                {
                    tourToStartDate[dbTour.upstream_identifier] = dbShow.date;
                }

                if (!tourToEndDate.ContainsKey(dbTour.upstream_identifier) ||
                    dbShow.date > tourToEndDate[dbTour.upstream_identifier])
                {
                    tourToEndDate[dbTour.upstream_identifier] = dbShow.date;
                }
            }

            await _setlistShowService.AddSongPlays(dbShow, dbSongs);
        }
Example #28
0
        public async Task <ImportStats> ProcessShows(Artist artist, ArtistUpstreamSource src, PerformContext ctx)
        {
            var stats = new ImportStats();

            var pages = 80;

            var prog     = ctx?.WriteProgressBar();
            var pageSize = 20;

            for (var currentPage = 1; currentPage <= pages; currentPage++)
            {
                var apiShows = await PhishinApiRequest <IEnumerable <PhishinShow> >("shows", ctx, "date", per_page : pageSize, page : currentPage);

                pages = apiShows.total_pages;

                var shows = apiShows.data.ToList();

                foreach (var(idx, show) in shows.Select((s, i) => (i, s)))
                {
                    try
                    {
                        await processShow(show);
                    }
                    catch (Exception e)
                    {
                        ctx?.WriteLine($"Error processing show (but continuing): {show.date} (id: {show.id})");
                        ctx?.LogException(e);
                    }

                    prog?.SetValue(100.0 * ((currentPage - 1) * pageSize + idx + 1) / apiShows.total_entries);
                }
            }

            async Task processShow(PhishinShow show)
            {
                using (var scope = new TransactionScope(TransactionScopeAsyncFlowOption.Enabled))
                {
                    var dbSource = existingSources.GetValue(show.id.ToString());

                    if (dbSource == null)
                    {
                        dbSource = await ProcessShow(stats, artist, show, src, new Source()
                        {
                            updated_at          = show.updated_at,
                            artist_id           = artist.id,
                            venue_id            = existingVenues[show.venue.id.ToString()].id,
                            display_date        = show.date,
                            upstream_identifier = show.id.ToString(),
                            is_soundboard       = show.sbd,
                            is_remaster         = show.remastered,
                            description         = "",
                            taper_notes         = show.taper_notes
                        }, ctx);

                        existingSources[dbSource.upstream_identifier] = dbSource;

                        stats.Created++;

                        stats.Created += (await linkService.AddLinksForSource(dbSource, new[] {
                            new Link
                            {
                                source_id = dbSource.id,
                                for_ratings = false,
                                for_source = true,
                                for_reviews = false,
                                upstream_source_id = src.upstream_source_id,
                                url = $"http://phish.in/{dbSource.display_date}",
                                label = "View on phish.in"
                            }
                        })).Count();
                    }
                    else if (show.updated_at > dbSource.updated_at)
                    {
                        dbSource.updated_at          = show.updated_at;
                        dbSource.venue_id            = existingVenues[show.venue.id.ToString()].id;
                        dbSource.display_date        = show.date;
                        dbSource.upstream_identifier = show.id.ToString();
                        dbSource.is_soundboard       = show.sbd;
                        dbSource.is_remaster         = show.remastered;
                        dbSource.description         = "";
                        dbSource.taper_notes         = show.taper_notes;

                        dbSource = await ProcessShow(stats, artist, show, src, dbSource, ctx);

                        existingSources[dbSource.upstream_identifier] = dbSource;

                        stats.Updated++;
                    }

                    scope.Complete();
                }
            }

            return(stats);
        }
        async Task <ImportStats> ProcessSetlist(Artist artist, Relisten.Vendor.SetlistFm.Setlist setlist)
        {
            var stats = new ImportStats();
            var now   = DateTime.UtcNow;

            // venue
            Venue dbVenue = existingVenues.GetValue(setlist.venue._iguanaUpstreamId);

            if (dbVenue == null)
            {
                var sc = new VenueWithShowCount()
                {
                    updated_at          = now,
                    artist_id           = artist.id,
                    name                = setlist.venue.name,
                    latitude            = setlist.venue.city.coords?.lat,
                    longitude           = setlist.venue.city.coords?.@long,
                    location            = $"{setlist.venue.city.name}, {setlist.venue.city.state}",
                    upstream_identifier = setlist.venue._iguanaUpstreamId,
                    slug                = Slugify(setlist.venue.name)
                };

                dbVenue = await _venueService.Save(sc);

                sc.id = dbVenue.id;

                existingVenues[dbVenue.upstream_identifier] = sc;

                stats.Created++;
            }

            // tour
            Tour dbTour = null;

            if (artist.features.tours)
            {
                var tour_upstream = setlist.tour?.name ?? "Not Part of a Tour";
                dbTour = existingTours.GetValue(tour_upstream);
                if (dbTour == null)
                {
                    dbTour = await _tourService.Save(new Tour()
                    {
                        updated_at          = now,
                        artist_id           = artist.id,
                        start_date          = null,
                        end_date            = null,
                        name                = tour_upstream,
                        slug                = Slugify(tour_upstream),
                        upstream_identifier = tour_upstream
                    });

                    existingTours[dbTour.upstream_identifier] = dbTour;

                    stats.Created++;
                }
            }

            // show
            var dbShow             = existingSetlistShows.GetValue(setlist.id);
            var date               = DateTime.ParseExact(setlist.eventDate, "dd-MM-yyyy", null);
            var setlistLastUpdated = setlist.lastUpdated;

            var shouldAddSongs = false;

            if (dbShow == null)
            {
                dbShow = await _setlistShowService.Save(new SetlistShow()
                {
                    artist_id           = artist.id,
                    updated_at          = setlistLastUpdated,
                    date                = date,
                    upstream_identifier = setlist.id,
                    venue_id            = dbVenue.id,
                    tour_id             = artist.features.tours ? dbTour?.id : null
                });

                existingSetlistShows[dbShow.upstream_identifier] = dbShow;

                stats.Created++;

                shouldAddSongs = true;
            }
            else if (setlistLastUpdated > dbShow.updated_at)
            {
                dbShow.artist_id  = artist.id;
                dbShow.updated_at = setlistLastUpdated;
                dbShow.date       = date;
                dbShow.venue_id   = dbVenue.id;
                dbShow.tour_id    = dbTour?.id;

                dbShow = await _setlistShowService.Save(dbShow);

                existingSetlistShows[dbShow.upstream_identifier] = dbShow;

                stats.Updated++;

                shouldAddSongs = true;
            }

            // setlist.fm doesn't provide much info about tours so we need to find the start
            // and end date ourselves.
            if (artist.features.tours &&
                (dbTour.start_date == null ||
                 dbTour.end_date == null ||
                 dbShow.date < dbTour.start_date ||
                 dbShow.date > dbTour.end_date))
            {
                if (!tourToStartDate.ContainsKey(dbTour.upstream_identifier) ||
                    dbShow.date < tourToStartDate[dbTour.upstream_identifier])
                {
                    tourToStartDate[dbTour.upstream_identifier] = dbShow.date;
                }

                if (!tourToEndDate.ContainsKey(dbTour.upstream_identifier) ||
                    dbShow.date > tourToEndDate[dbTour.upstream_identifier])
                {
                    tourToEndDate[dbTour.upstream_identifier] = dbShow.date;
                }
            }

            if (shouldAddSongs)
            {
                var songs = setlist.sets.set.
                            SelectMany(set => set.song).
                            Select(song => new { Name = song.name, Slug = Slugify(song.name) }).
                            GroupBy(song => song.Slug).
                            Select(grp => grp.First()).
                            ToList();

                var dbSongs = existingSetlistSongs.
                              Where(kvp => songs.Select(song => song.Slug).Contains(kvp.Key)).
                              Select(kvp => kvp.Value).
                              ToList();

                if (songs.Count != dbSongs.Count)
                {
                    var newSongs = songs.
                                   Where(song => dbSongs.Find(dbSong => dbSong.slug == song.Slug) == null).
                                   Select(song => new SetlistSong()
                    {
                        artist_id           = artist.id,
                        name                = song.Name,
                        slug                = song.Slug,
                        updated_at          = now,
                        upstream_identifier = song.Slug
                    }).
                                   ToList();

                    var justAdded = await _setlistSongService.InsertAll(artist, newSongs);

                    dbSongs.AddRange(justAdded);
                    stats.Created += newSongs.Count;

                    foreach (var justAddedSong in justAdded)
                    {
                        existingSetlistSongs[justAddedSong.upstream_identifier] = justAddedSong;
                    }
                }

                stats += await _setlistShowService.UpdateSongPlays(dbShow, dbSongs);
            }

            return(stats);
        }
Example #30
0
        private async Task <ImportStats> ProcessSource(Artist artist, ArtistUpstreamSource src, Source dbSource, PerformContext ctx)
        {
            var stats = new ImportStats();

            var ratings = await ScrapePhishNetForSource(dbSource, ctx);

            var dirty = false;

            if (dbSource.num_ratings != ratings.RatingVotesCast)
            {
                dbSource.num_ratings = ratings.RatingVotesCast;
                dbSource.avg_rating  = ratings.RatingAverage * 2.0;

                dirty = true;
            }

            if (dbSource.num_reviews != ratings.NumberOfReviewsWritten)
            {
                var reviewsTask = GetPhishNetApiReviews(dbSource, ctx);
                var setlistTask = GetPhishNetApiSetlist(dbSource, ctx);

                await Task.WhenAll(reviewsTask, setlistTask);

                var dbReviews = reviewsTask.Result.Select(rev =>
                {
                    return(new SourceReview()
                    {
                        rating = null,
                        title = null,
                        review = rev.review,
                        author = rev.author,
                        updated_at = DateTimeOffset.FromUnixTimeSeconds(rev.tstamp).UtcDateTime
                    });
                }).ToList();

                dbSource.num_reviews = dbReviews.Count();
                dbSource.description = setlistTask.Result.setlistnotes + "\n\n\n" + setlistTask.Result.setlistdata;

                dirty = true;

                await ReplaceSourceReviews(stats, dbSource, dbReviews);
            }

            if (dirty)
            {
                await _sourceService.Save(dbSource);

                stats.Updated++;
            }

            stats.Created += (await linkService.AddLinksForSource(dbSource, new[]
            {
                new Link
                {
                    source_id = dbSource.id,
                    for_ratings = true,
                    for_source = false,
                    for_reviews = true,
                    upstream_source_id = src.upstream_source_id,
                    url = PhishNetUrlForSource(dbSource),
                    label = "View on phish.net"
                }
            })).Count();


            return(stats);
        }