private async Task ProcessShow(ImportStats stats, Artist artist, ArtistUpstreamSource upstreamSrc, string showDate, string sourceName, IList <PanicStream.PanicStreamTrack> sourceTracks, PerformContext ctx)
{
var upstreamId = sourceName;
var dbSource = existingSources.GetValue(upstreamId);
var panicUpdatedAt = sourceTracks
.Where(t => t.System.ParsedModificationTime.HasValue)
.Max(t => t.System.ParsedModificationTime.Value);
if (dbSource != null && dbSource.updated_at <= panicUpdatedAt)
{
return;
}
var isUpdate = dbSource != null;
var src = new Source
{
artist_id = artist.id,
display_date = showDate,
is_soundboard = false,
is_remaster = false,
has_jamcharts = false,
avg_rating = 0,
num_reviews = 0,
avg_rating_weighted = 0,
upstream_identifier = upstreamId,
taper_notes = "",
updated_at = panicUpdatedAt
};
if (isUpdate)
{
src.id = dbSource.id;
}
dbSource = await _sourceService.Save(src);
existingSources[dbSource.upstream_identifier] = dbSource;
if (isUpdate)
{
stats.Updated++;
}
else
{
stats.Created++;
stats.Created += (await linkService.AddLinksForSource(dbSource, new[]
{
new Link
{
source_id = dbSource.id,
for_ratings = false,
for_source = true,
for_reviews = false,
upstream_source_id = upstreamSrc.upstream_source_id,
url = $"https://www.panicstream.com/vault/widespread-panic/{dbSource.display_date.Substring(0, 4)}-streams/",
label = "View show page on panicstream.com"
}
})).Count();
}
var dbSet = await _sourceSetService.Update(dbSource, new SourceSet
{
source_id = dbSource.id,
index = 0,
is_encore = false,
name = "Default Set",
updated_at = panicUpdatedAt
});
stats.Created++;
var trackIndex = 0;
var mp3s = sourceTracks
.OrderBy(t => t.FileName)
.Select(t =>
{
var trackName = t.FileName
.Replace(".mp3", "")
.Replace(".MP3", "")
.Replace(".M4A", "")
.Replace(".m4a", "")
.Trim();
var cleanedTrackName = Regex.Replace(trackName, @"(wsp[0-9-]+d\d+t\d+\.)|(^\d+ ?-? ?)", "").Trim();
if (cleanedTrackName.Length != 0)
{
trackName = cleanedTrackName;
}
trackIndex++;
return(new SourceTrack
{
source_id = dbSource.id,
source_set_id = dbSet.id,
track_position = trackIndex,
duration = ((int?)t.Composite?.CalculatedDuration.TotalSeconds ?? (int?)0).Value,
title = trackName,
slug = SlugifyTrack(trackName),
mp3_url = t.AbsoluteUrl(_configuration["PANIC_KEY"]),
updated_at = panicUpdatedAt,
artist_id = artist.id
});
});
ResetTrackSlugCounts();
await _sourceTrackService.InsertAll(dbSource, mp3s);
stats.Created += mp3s.Count();
}
public async Task <ImportStats> ProcessShows(Artist artist, ArtistUpstreamSource src, PerformContext ctx)
{
var stats = new ImportStats();
var pages = 80;
var prog = ctx?.WriteProgressBar();
var pageSize = 20;
for (var currentPage = 1; currentPage <= pages; currentPage++)
{
var apiShows = await PhishinApiRequest <IEnumerable <PhishinShow> >("shows", ctx, "date", per_page : pageSize, page : currentPage);
pages = apiShows.total_pages;
var shows = apiShows.data.ToList();
foreach (var(idx, show) in shows.Select((s, i) => (i, s)))
{
try
{
await processShow(show);
}
catch (Exception e)
{
ctx?.WriteLine($"Error processing show (but continuing): {show.date} (id: {show.id})");
ctx?.LogException(e);
}
prog?.SetValue(100.0 * ((currentPage - 1) * pageSize + idx + 1) / apiShows.total_entries);
}
}
async Task processShow(PhishinShow show)
{
using (var scope = new TransactionScope(TransactionScopeAsyncFlowOption.Enabled))
{
var dbSource = existingSources.GetValue(show.id.ToString());
if (dbSource == null)
{
dbSource = await ProcessShow(stats, artist, show, src, new Source()
{
updated_at = show.updated_at,
artist_id = artist.id,
venue_id = existingVenues[show.venue.id.ToString()].id,
display_date = show.date,
upstream_identifier = show.id.ToString(),
is_soundboard = show.sbd,
is_remaster = show.remastered,
description = "",
taper_notes = show.taper_notes
}, ctx);
existingSources[dbSource.upstream_identifier] = dbSource;
stats.Created++;
stats.Created += (await linkService.AddLinksForSource(dbSource, new[] {
new Link
{
source_id = dbSource.id,
for_ratings = false,
for_source = true,
for_reviews = false,
upstream_source_id = src.upstream_source_id,
url = $"http://phish.in/{dbSource.display_date}",
label = "View on phish.in"
}
})).Count();
}
else if (show.updated_at > dbSource.updated_at)
{
dbSource.updated_at = show.updated_at;
dbSource.venue_id = existingVenues[show.venue.id.ToString()].id;
dbSource.display_date = show.date;
dbSource.upstream_identifier = show.id.ToString();
dbSource.is_soundboard = show.sbd;
dbSource.is_remaster = show.remastered;
dbSource.description = "";
dbSource.taper_notes = show.taper_notes;
dbSource = await ProcessShow(stats, artist, show, src, dbSource, ctx);
existingSources[dbSource.upstream_identifier] = dbSource;
stats.Updated++;
}
scope.Complete();
}
}
return(stats);
}
private async Task <ImportStats> ProcessSource(Artist artist, ArtistUpstreamSource src, Source dbSource, PerformContext ctx)
{
var stats = new ImportStats();
var ratings = await ScrapePhishNetForSource(dbSource, ctx);
var dirty = false;
if (dbSource.num_ratings != ratings.RatingVotesCast)
{
dbSource.num_ratings = ratings.RatingVotesCast;
dbSource.avg_rating = ratings.RatingAverage * 2.0;
dirty = true;
}
if (dbSource.num_reviews != ratings.NumberOfReviewsWritten)
{
var reviewsTask = GetPhishNetApiReviews(dbSource, ctx);
var setlistTask = GetPhishNetApiSetlist(dbSource, ctx);
await Task.WhenAll(reviewsTask, setlistTask);
var dbReviews = reviewsTask.Result.Select(rev =>
{
return(new SourceReview()
{
rating = null,
title = null,
review = rev.review,
author = rev.author,
updated_at = DateTimeOffset.FromUnixTimeSeconds(rev.tstamp).UtcDateTime
});
}).ToList();
dbSource.num_reviews = dbReviews.Count();
dbSource.description = setlistTask.Result.setlistnotes + "\n\n\n" + setlistTask.Result.setlistdata;
dirty = true;
await ReplaceSourceReviews(stats, dbSource, dbReviews);
}
if (dirty)
{
await _sourceService.Save(dbSource);
stats.Updated++;
}
stats.Created += (await linkService.AddLinksForSource(dbSource, new[]
{
new Link
{
source_id = dbSource.id,
for_ratings = true,
for_source = false,
for_reviews = true,
upstream_source_id = src.upstream_source_id,
url = PhishNetUrlForSource(dbSource),
label = "View on phish.net"
}
})).Count();
return(stats);
}
private async Task <ImportStats> ImportSingleIdentifier(
Artist artist,
Source dbSource,
Relisten.Vendor.ArchiveOrg.SearchDoc searchDoc,
Relisten.Vendor.ArchiveOrg.Metadata.RootObject detailsRoot,
ArtistUpstreamSource upstreamSrc,
string properDisplayDate,
PerformContext ctx
)
{
var stats = new ImportStats();
var upstream_identifier = searchDoc.identifier;
var isUpdate = dbSource != null;
var meta = detailsRoot.metadata;
var mp3Files = detailsRoot.files?.Where(file => file?.format == "VBR MP3");
var flacFiles = detailsRoot.files?.Where(file => file?.format == "Flac" || file?.format == "24bit Flac");
if (mp3Files == null || mp3Files.Count() == 0)
{
ctx?.WriteLine("\tNo VBR MP3 files found for {0}", searchDoc.identifier);
throw new NoVBRMp3FilesException();
}
var dbReviews = detailsRoot.reviews == null
? new List <SourceReview>()
: detailsRoot.reviews.Select(rev =>
{
return(new SourceReview()
{
rating = rev.stars * 2, // scale to out of 10
title = rev.reviewtitle,
review = rev.reviewbody?.Replace("Â", "") ?? "",
author = rev.reviewer,
updated_at = rev.reviewdate
});
}).ToList();
Venue dbVenue = null;
if (artist.features.per_source_venues)
{
var venueName = String.IsNullOrEmpty(meta.venue) ? meta.coverage : meta.venue;
if (String.IsNullOrEmpty(venueName))
{
venueName = "Unknown Venue";
}
var venueUpstreamId = venueName + (String.IsNullOrEmpty(meta.coverage) ? "blank coverage" : meta.coverage);
dbVenue = await _venueService.ForUpstreamIdentifier(artist, venueUpstreamId);
if (dbVenue == null)
{
dbVenue = await _venueService.Save(new Venue()
{
artist_id = artist.id,
name = venueName,
location = String.IsNullOrEmpty(meta.coverage) ? "Unknown Location" : meta.coverage,
upstream_identifier = venueUpstreamId,
slug = Slugify(venueName),
updated_at = searchDoc._iguana_updated_at
});
}
}
if (isUpdate)
{
var src = CreateSourceForMetadata(artist, detailsRoot, searchDoc, properDisplayDate);
src.id = dbSource.id;
src.venue_id = dbVenue.id;
dbSource = await _sourceService.Save(src);
dbSource.venue = dbVenue;
stats.Updated++;
stats.Created += (await ReplaceSourceReviews(dbSource, dbReviews)).Count();
}
else
{
dbSource = await _sourceService.Save(CreateSourceForMetadata(artist, detailsRoot, searchDoc, properDisplayDate, dbVenue));
stats.Created++;
existingSources[dbSource.upstream_identifier] = dbSource;
stats.Created += (await ReplaceSourceReviews(dbSource, dbReviews)).Count();
}
stats.Created += (await linkService.AddLinksForSource(dbSource, LinksForSource(artist, dbSource, upstreamSrc))).Count();
var dbSet = (await _sourceSetService.UpdateAll(dbSource, new[] { CreateSetForSource(dbSource) })).First();
stats.Created++;
var flacTracksByName = flacFiles.GroupBy(f => f.name).ToDictionary(g => g.Key, g => g.First());
var dbTracks = CreateSourceTracksForFiles(artist, dbSource, meta, mp3Files, flacTracksByName, dbSet);
stats.Created += (await _sourceTrackService.InsertAll(dbSource, dbTracks)).Count();
ResetTrackSlugCounts();
return(stats);
}