private async Task ProcessShow(ImportStats stats, Artist artist, ArtistUpstreamSource upstreamSrc, string showDate, string sourceName, IList <PanicStream.PanicStreamTrack> sourceTracks, PerformContext ctx) { var upstreamId = sourceName; var dbSource = existingSources.GetValue(upstreamId); var panicUpdatedAt = sourceTracks .Where(t => t.System.ParsedModificationTime.HasValue) .Max(t => t.System.ParsedModificationTime.Value); if (dbSource != null && dbSource.updated_at <= panicUpdatedAt) { return; } var isUpdate = dbSource != null; var src = new Source { artist_id = artist.id, display_date = showDate, is_soundboard = false, is_remaster = false, has_jamcharts = false, avg_rating = 0, num_reviews = 0, avg_rating_weighted = 0, upstream_identifier = upstreamId, taper_notes = "", updated_at = panicUpdatedAt }; if (isUpdate) { src.id = dbSource.id; } dbSource = await _sourceService.Save(src); existingSources[dbSource.upstream_identifier] = dbSource; if (isUpdate) { stats.Updated++; } else { stats.Created++; stats.Created += (await linkService.AddLinksForSource(dbSource, new[] { new Link { source_id = dbSource.id, for_ratings = false, for_source = true, for_reviews = false, upstream_source_id = upstreamSrc.upstream_source_id, url = $"https://www.panicstream.com/vault/widespread-panic/{dbSource.display_date.Substring(0, 4)}-streams/", label = "View show page on panicstream.com" } })).Count(); } var dbSet = await _sourceSetService.Update(dbSource, new SourceSet { source_id = dbSource.id, index = 0, is_encore = false, name = "Default Set", updated_at = panicUpdatedAt }); stats.Created++; var trackIndex = 0; var mp3s = sourceTracks .OrderBy(t => t.FileName) .Select(t => { var trackName = t.FileName .Replace(".mp3", "") .Replace(".MP3", "") .Replace(".M4A", "") .Replace(".m4a", "") .Trim(); var cleanedTrackName = Regex.Replace(trackName, @"(wsp[0-9-]+d\d+t\d+\.)|(^\d+ ?-? ?)", "").Trim(); if (cleanedTrackName.Length != 0) { trackName = cleanedTrackName; } trackIndex++; return(new SourceTrack { source_id = dbSource.id, source_set_id = dbSet.id, track_position = trackIndex, duration = ((int?)t.Composite?.CalculatedDuration.TotalSeconds ?? (int?)0).Value, title = trackName, slug = SlugifyTrack(trackName), mp3_url = t.AbsoluteUrl(_configuration["PANIC_KEY"]), updated_at = panicUpdatedAt, artist_id = artist.id }); }); ResetTrackSlugCounts(); await _sourceTrackService.InsertAll(dbSource, mp3s); stats.Created += mp3s.Count(); }
public async Task <ImportStats> ProcessShows(Artist artist, ArtistUpstreamSource src, PerformContext ctx) { var stats = new ImportStats(); var pages = 80; var prog = ctx?.WriteProgressBar(); var pageSize = 20; for (var currentPage = 1; currentPage <= pages; currentPage++) { var apiShows = await PhishinApiRequest <IEnumerable <PhishinShow> >("shows", ctx, "date", per_page : pageSize, page : currentPage); pages = apiShows.total_pages; var shows = apiShows.data.ToList(); foreach (var(idx, show) in shows.Select((s, i) => (i, s))) { try { await processShow(show); } catch (Exception e) { ctx?.WriteLine($"Error processing show (but continuing): {show.date} (id: {show.id})"); ctx?.LogException(e); } prog?.SetValue(100.0 * ((currentPage - 1) * pageSize + idx + 1) / apiShows.total_entries); } } async Task processShow(PhishinShow show) { using (var scope = new TransactionScope(TransactionScopeAsyncFlowOption.Enabled)) { var dbSource = existingSources.GetValue(show.id.ToString()); if (dbSource == null) { dbSource = await ProcessShow(stats, artist, show, src, new Source() { updated_at = show.updated_at, artist_id = artist.id, venue_id = existingVenues[show.venue.id.ToString()].id, display_date = show.date, upstream_identifier = show.id.ToString(), is_soundboard = show.sbd, is_remaster = show.remastered, description = "", taper_notes = show.taper_notes }, ctx); existingSources[dbSource.upstream_identifier] = dbSource; stats.Created++; stats.Created += (await linkService.AddLinksForSource(dbSource, new[] { new Link { source_id = dbSource.id, for_ratings = false, for_source = true, for_reviews = false, upstream_source_id = src.upstream_source_id, url = $"http://phish.in/{dbSource.display_date}", label = "View on phish.in" } })).Count(); } else if (show.updated_at > dbSource.updated_at) { dbSource.updated_at = show.updated_at; dbSource.venue_id = existingVenues[show.venue.id.ToString()].id; dbSource.display_date = show.date; dbSource.upstream_identifier = show.id.ToString(); dbSource.is_soundboard = show.sbd; dbSource.is_remaster = show.remastered; dbSource.description = ""; dbSource.taper_notes = show.taper_notes; dbSource = await ProcessShow(stats, artist, show, src, dbSource, ctx); existingSources[dbSource.upstream_identifier] = dbSource; stats.Updated++; } scope.Complete(); } } return(stats); }
private async Task <ImportStats> ProcessSource(Artist artist, ArtistUpstreamSource src, Source dbSource, PerformContext ctx) { var stats = new ImportStats(); var ratings = await ScrapePhishNetForSource(dbSource, ctx); var dirty = false; if (dbSource.num_ratings != ratings.RatingVotesCast) { dbSource.num_ratings = ratings.RatingVotesCast; dbSource.avg_rating = ratings.RatingAverage * 2.0; dirty = true; } if (dbSource.num_reviews != ratings.NumberOfReviewsWritten) { var reviewsTask = GetPhishNetApiReviews(dbSource, ctx); var setlistTask = GetPhishNetApiSetlist(dbSource, ctx); await Task.WhenAll(reviewsTask, setlistTask); var dbReviews = reviewsTask.Result.Select(rev => { return(new SourceReview() { rating = null, title = null, review = rev.review, author = rev.author, updated_at = DateTimeOffset.FromUnixTimeSeconds(rev.tstamp).UtcDateTime }); }).ToList(); dbSource.num_reviews = dbReviews.Count(); dbSource.description = setlistTask.Result.setlistnotes + "\n\n\n" + setlistTask.Result.setlistdata; dirty = true; await ReplaceSourceReviews(stats, dbSource, dbReviews); } if (dirty) { await _sourceService.Save(dbSource); stats.Updated++; } stats.Created += (await linkService.AddLinksForSource(dbSource, new[] { new Link { source_id = dbSource.id, for_ratings = true, for_source = false, for_reviews = true, upstream_source_id = src.upstream_source_id, url = PhishNetUrlForSource(dbSource), label = "View on phish.net" } })).Count(); return(stats); }
private async Task <ImportStats> ImportSingleIdentifier( Artist artist, Source dbSource, Relisten.Vendor.ArchiveOrg.SearchDoc searchDoc, Relisten.Vendor.ArchiveOrg.Metadata.RootObject detailsRoot, ArtistUpstreamSource upstreamSrc, string properDisplayDate, PerformContext ctx ) { var stats = new ImportStats(); var upstream_identifier = searchDoc.identifier; var isUpdate = dbSource != null; var meta = detailsRoot.metadata; var mp3Files = detailsRoot.files?.Where(file => file?.format == "VBR MP3"); var flacFiles = detailsRoot.files?.Where(file => file?.format == "Flac" || file?.format == "24bit Flac"); if (mp3Files == null || mp3Files.Count() == 0) { ctx?.WriteLine("\tNo VBR MP3 files found for {0}", searchDoc.identifier); throw new NoVBRMp3FilesException(); } var dbReviews = detailsRoot.reviews == null ? new List <SourceReview>() : detailsRoot.reviews.Select(rev => { return(new SourceReview() { rating = rev.stars * 2, // scale to out of 10 title = rev.reviewtitle, review = rev.reviewbody?.Replace("Â", "") ?? "", author = rev.reviewer, updated_at = rev.reviewdate }); }).ToList(); Venue dbVenue = null; if (artist.features.per_source_venues) { var venueName = String.IsNullOrEmpty(meta.venue) ? meta.coverage : meta.venue; if (String.IsNullOrEmpty(venueName)) { venueName = "Unknown Venue"; } var venueUpstreamId = venueName + (String.IsNullOrEmpty(meta.coverage) ? "blank coverage" : meta.coverage); dbVenue = await _venueService.ForUpstreamIdentifier(artist, venueUpstreamId); if (dbVenue == null) { dbVenue = await _venueService.Save(new Venue() { artist_id = artist.id, name = venueName, location = String.IsNullOrEmpty(meta.coverage) ? "Unknown Location" : meta.coverage, upstream_identifier = venueUpstreamId, slug = Slugify(venueName), updated_at = searchDoc._iguana_updated_at }); } } if (isUpdate) { var src = CreateSourceForMetadata(artist, detailsRoot, searchDoc, properDisplayDate); src.id = dbSource.id; src.venue_id = dbVenue.id; dbSource = await _sourceService.Save(src); dbSource.venue = dbVenue; stats.Updated++; stats.Created += (await ReplaceSourceReviews(dbSource, dbReviews)).Count(); } else { dbSource = await _sourceService.Save(CreateSourceForMetadata(artist, detailsRoot, searchDoc, properDisplayDate, dbVenue)); stats.Created++; existingSources[dbSource.upstream_identifier] = dbSource; stats.Created += (await ReplaceSourceReviews(dbSource, dbReviews)).Count(); } stats.Created += (await linkService.AddLinksForSource(dbSource, LinksForSource(artist, dbSource, upstreamSrc))).Count(); var dbSet = (await _sourceSetService.UpdateAll(dbSource, new[] { CreateSetForSource(dbSource) })).First(); stats.Created++; var flacTracksByName = flacFiles.GroupBy(f => f.name).ToDictionary(g => g.Key, g => g.First()); var dbTracks = CreateSourceTracksForFiles(artist, dbSource, meta, mp3Files, flacTracksByName, dbSet); stats.Created += (await _sourceTrackService.InsertAll(dbSource, dbTracks)).Count(); ResetTrackSlugCounts(); return(stats); }